From 735147d4d6434ba236571be3ec88f2ec53ea6554 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Wed, 4 Mar 2020 10:37:54 -0700 Subject: Pass some env vars that affect msvs tool setup. MSCommon already makes sure a few variables are propagated before calling the the dev env setup script (vcvarsall.bat or relative). This change adds two more: VSCMD_DEBUG enables VsDevCmd.bat (which is eventually called) to log the environment before and after setup, and VSCMD_SKIP_SENDTELEMETRY, if set, inhibits the new (VS 2019) functionality to send telemetry information. The change is motivated by the telemetry code generating undecodable output for certain locales. This should allow suppressing it, but doesn't really fix the underlying problem that we're not handling text right on Windows (on Py2 this would not have failed, but that was an error as well - it should have). Signed-off-by: Mats Wichmann --- src/CHANGES.txt | 2 ++ src/engine/SCons/Tool/MSCommon/common.py | 27 +++++++++++++++++---------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/CHANGES.txt b/src/CHANGES.txt index 514ecf7..958a475 100755 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -84,6 +84,8 @@ From Rob Boehne - Fixed bug where changing TEXTFILESUFFIX would cause Substfile() to rebuild. (Github Issue #3540) - Script/Main.py now uses importlib instead of imp module. - Drop some Python 2-isms. + - Pass on VSCMD_DEBUG and VSCMD_SKIP_SENDTELEMETRY to msvc tool setup + if set in environment. RELEASE 3.1.2 - Mon, 17 Dec 2019 02:06:27 +0000 diff --git a/src/engine/SCons/Tool/MSCommon/common.py b/src/engine/SCons/Tool/MSCommon/common.py index 7f832c6..422d897 100644 --- a/src/engine/SCons/Tool/MSCommon/common.py +++ b/src/engine/SCons/Tool/MSCommon/common.py @@ -185,16 +185,21 @@ def get_output(vcbat, args=None, env=None): # Create a blank environment, for use in launching the tools env = SCons.Environment.Environment(tools=[]) - # TODO: This is a hard-coded list of the variables that (may) need - # to be imported from os.environ[] for v[sc]*vars*.bat file - # execution to work. This list should really be either directly - # controlled by vc.py, or else derived from the common_tools_var - # settings in vs.py. + # TODO: Hard-coded list of the variables that (may) need to be + # imported from os.environ[] for the chain of development batch + # files to execute correctly. One call to vcvars*.bat may + # end up running a dozen or more scripts, changes not only with + # each release but with what is installed at the time. We think + # in modern installations most are set along the way and don't + # need to be picked from the env, but include these for safety's sake. + # Any VSCMD variables definitely are picked from the env and + # control execution in interesting ways. + # Note these really should be unified - either controlled by vs.py, + # or synced with the the common_tools_var # settings in vs.py. vs_vc_vars = [ - 'COMSPEC', - # VS100 and VS110: Still set, but modern MSVC setup scripts will - # discard these if registry has values. However Intel compiler setup - # script still requires these as of 2013/2014. + 'COMSPEC', # path to "shell" + 'VS160COMNTOOLS', # path to common tools for given version + 'VS150COMNTOOLS', 'VS140COMNTOOLS', 'VS120COMNTOOLS', 'VS110COMNTOOLS', @@ -204,6 +209,8 @@ def get_output(vcbat, args=None, env=None): 'VS71COMNTOOLS', 'VS70COMNTOOLS', 'VS60COMNTOOLS', + 'VSCMD_DEBUG', # enable logging and other debug aids + 'VSCMD_SKIP_SENDTELEMETRY', ] env['ENV'] = normalize_env(env['ENV'], vs_vc_vars, force=False) @@ -237,7 +244,7 @@ def get_output(vcbat, args=None, env=None): if stderr: # TODO: find something better to do with stderr; # this at least prevents errors from getting swallowed. - sys.stderr.write(stderr) + sys.stderr.write(stderr.decode("mbcs")) if popen.wait() != 0: raise IOError(stderr.decode("mbcs")) -- cgit v0.12 From b18601eb0e057afcbdbc92ff516742767361f192 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Tue, 17 Mar 2020 08:22:08 -0600 Subject: [PR #3576] add powershell path We pass a very minimal PATH to the call to vcvars*. Apparently there are circumstances where not having Powershell in it can cause failures (relates to the same telemetry call that is the subject of this PR). Signed-off-by: Mats Wichmann --- src/CHANGES.txt | 5 +++-- src/engine/SCons/Tool/MSCommon/common.py | 25 +++++++++++++++---------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/CHANGES.txt b/src/CHANGES.txt index 958a475..2147f49 100755 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -84,8 +84,9 @@ From Rob Boehne - Fixed bug where changing TEXTFILESUFFIX would cause Substfile() to rebuild. (Github Issue #3540) - Script/Main.py now uses importlib instead of imp module. - Drop some Python 2-isms. - - Pass on VSCMD_DEBUG and VSCMD_SKIP_SENDTELEMETRY to msvc tool setup - if set in environment. + - MSVC updates: pass on VSCMD_DEBUG and VSCMD_SKIP_SENDTELEMETRY to msvc + tool setup if set in environment. Add powershell to default env + (used to call telemetry script). RELEASE 3.1.2 - Mon, 17 Dec 2019 02:06:27 +0000 diff --git a/src/engine/SCons/Tool/MSCommon/common.py b/src/engine/SCons/Tool/MSCommon/common.py index 422d897..c066fd8 100644 --- a/src/engine/SCons/Tool/MSCommon/common.py +++ b/src/engine/SCons/Tool/MSCommon/common.py @@ -156,15 +156,14 @@ def normalize_env(env, keys, force=False): if k in os.environ and (force or k not in normenv): normenv[k] = os.environ[k] - # This shouldn't be necessary, since the default environment should include system32, - # but keep this here to be safe, since it's needed to find reg.exe which the MSVC - # bat scripts use. - sys32_dir = os.path.join(os.environ.get("SystemRoot", - os.environ.get("windir", r"C:\Windows\system32")), - "System32") - - if sys32_dir not in normenv['PATH']: - normenv['PATH'] = normenv['PATH'] + os.pathsep + sys32_dir + # add some things to PATH to prevent problems: + # Shouldn't be necessary to add system32, since the default environment + # should include it, but keep this here to be safe (needed for reg.exe) + sys32_dir = os.path.join( + os.environ.get("SystemRoot", os.environ.get("windir", r"C:\Windows")), "System32" +) + if sys32_dir not in normenv["PATH"]: + normenv["PATH"] = normenv["PATH"] + os.pathsep + sys32_dir # Without Wbem in PATH, vcvarsall.bat has a "'wmic' is not recognized" # error starting with Visual Studio 2017, although the script still @@ -173,8 +172,14 @@ def normalize_env(env, keys, force=False): if sys32_wbem_dir not in normenv['PATH']: normenv['PATH'] = normenv['PATH'] + os.pathsep + sys32_wbem_dir - debug("PATH: %s" % normenv['PATH']) + # Without Powershell in PATH, an internal call to a telemetry + # function (starting with a VS2019 update) can fail + # Note can also set VSCMD_SKIP_SENDTELEMETRY to avoid this. + sys32_ps_dir = os.path.join(sys32_dir, r'WindowsPowerShell\1.0') + if sys32_ps_dir not in normenv['PATH']: + normenv['PATH'] = normenv['PATH'] + os.pathsep + sys32_ps_dir + debug("PATH: %s" % normenv['PATH']) return normenv -- cgit v0.12 From ddc4bd8a969853b27b8275fab81a6fafc9bff732 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Tue, 17 Mar 2020 09:59:52 -0600 Subject: [PR #3576] fix typo in powershell path Signed-off-by: Mats Wichmann --- src/engine/SCons/Tool/MSCommon/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engine/SCons/Tool/MSCommon/common.py b/src/engine/SCons/Tool/MSCommon/common.py index c066fd8..b7203a9 100644 --- a/src/engine/SCons/Tool/MSCommon/common.py +++ b/src/engine/SCons/Tool/MSCommon/common.py @@ -175,7 +175,7 @@ def normalize_env(env, keys, force=False): # Without Powershell in PATH, an internal call to a telemetry # function (starting with a VS2019 update) can fail # Note can also set VSCMD_SKIP_SENDTELEMETRY to avoid this. - sys32_ps_dir = os.path.join(sys32_dir, r'WindowsPowerShell\1.0') + sys32_ps_dir = os.path.join(sys32_dir, r'WindowsPowerShell\v1.0') if sys32_ps_dir not in normenv['PATH']: normenv['PATH'] = normenv['PATH'] + os.pathsep + sys32_ps_dir -- cgit v0.12 From 67aea476a01613edc6861775fbbc59b642604bbb Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Thu, 19 Mar 2020 08:46:41 -0600 Subject: [PR #3576] change decoding to "oem" in vcvars* runner Signed-off-by: Mats Wichmann --- src/engine/SCons/Tool/MSCommon/common.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/engine/SCons/Tool/MSCommon/common.py b/src/engine/SCons/Tool/MSCommon/common.py index b7203a9..b90c25a 100644 --- a/src/engine/SCons/Tool/MSCommon/common.py +++ b/src/engine/SCons/Tool/MSCommon/common.py @@ -246,21 +246,28 @@ def get_output(vcbat, args=None, env=None): # debug('get_output():stdout:%s'%stdout) # debug('get_output():stderr:%s'%stderr) + # Ongoing problems getting non-corrupted text led to this + # changing to "oem" from "mbcs" - the scripts run presumably + # attached to a console, so some particular rules apply. if stderr: # TODO: find something better to do with stderr; # this at least prevents errors from getting swallowed. - sys.stderr.write(stderr.decode("mbcs")) + sys.stderr.write(stderr.decode("oem")) if popen.wait() != 0: - raise IOError(stderr.decode("mbcs")) + raise IOError(stderr.decode("oem")) - output = stdout.decode("mbcs") - return output + return stdout.decode("oem") -KEEPLIST = ("INCLUDE", "LIB", "LIBPATH", "PATH", 'VSCMD_ARG_app_plat', - 'VCINSTALLDIR', # needed by clang -VS 2017 and newer - 'VCToolsInstallDir', # needed by clang - VS 2015 and older - ) +KEEPLIST = ( + "INCLUDE", + "LIB", + "LIBPATH", + "PATH", + "VSCMD_ARG_app_plat", + "VCINSTALLDIR", # needed by clang -VS 2017 and newer + "VCToolsInstallDir", # needed by clang - VS 2015 and older +) def parse_output(output, keep=KEEPLIST): -- cgit v0.12 From bb46401bfa7fbecdbc7faac11fce10fab985d9fd Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Tue, 7 Apr 2020 09:00:40 -0600 Subject: Manpage example tweaks [ci skip] A set of minor changes to various extra files (i.e. not scons.xml itself). Mainly this reformats some example code when it looked bad in a fixed-width version (pdf or Linux-style manpage). Also: additional wording on default environment (included here because file was being modified anyway), and define an entity for default env. in doc/scons.mod. Commits changes to two generated .mod files to make sure entities list is correct even if even if a full regen of these files (bin/docs-update-generated) is not done. Signed-off-by: Mats Wichmann --- doc/generated/functions.mod | 4 --- doc/generated/tools.mod | 6 +++-- doc/scons.mod | 2 ++ src/engine/SCons/Defaults.xml | 11 ++++---- src/engine/SCons/Environment.xml | 7 ++--- src/engine/SCons/Tool/applelink.xml | 6 ++--- src/engine/SCons/Tool/jar.xml | 2 +- src/engine/SCons/Tool/javac.xml | 2 +- src/engine/SCons/Tool/javah.xml | 17 +++++++------ src/engine/SCons/Tool/msvs.xml | 6 ++++- src/engine/SCons/Tool/packaging/__init__.xml | 38 +++++++++++++++------------- 11 files changed, 54 insertions(+), 47 deletions(-) diff --git a/doc/generated/functions.mod b/doc/generated/functions.mod index 3d49229..47d2be4 100644 --- a/doc/generated/functions.mod +++ b/doc/generated/functions.mod @@ -80,7 +80,6 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. SetDefault"> SetOption"> SideEffect"> -SourceCode"> Split"> subst"> Tag"> @@ -161,7 +160,6 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. env.SetDefault"> env.SetOption"> env.SideEffect"> -env.SourceCode"> env.Split"> env.subst"> env.Tag"> @@ -252,7 +250,6 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. SetDefault"> SetOption"> SideEffect"> -SourceCode"> Split"> subst"> Tag"> @@ -333,7 +330,6 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. env.SetDefault"> env.SetOption"> env.SideEffect"> -env.SourceCode"> env.Split"> env.subst"> env.Tag"> diff --git a/doc/generated/tools.mod b/doc/generated/tools.mod index 1209d74..3f8e22a 100644 --- a/doc/generated/tools.mod +++ b/doc/generated/tools.mod @@ -78,11 +78,12 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. mwcc"> mwld"> nasm"> -Packaging"> packaging"> +Packaging"> pdf"> pdflatex"> pdftex"> +python"> qt"> rmic"> rpcgen"> @@ -186,11 +187,12 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. mwcc"> mwld"> nasm"> -Packaging"> packaging"> +Packaging"> pdf"> pdflatex"> pdftex"> +python"> qt"> rmic"> rpcgen"> diff --git a/doc/scons.mod b/doc/scons.mod index bc63a42..006107e 100644 --- a/doc/scons.mod +++ b/doc/scons.mod @@ -447,6 +447,8 @@ Construction environments"> construction environment"> construction environments"> +Default Environment"> +default environment"> Construction Variable"> Construction Variables"> diff --git a/src/engine/SCons/Defaults.xml b/src/engine/SCons/Defaults.xml index 22f46fe..06fad1c 100644 --- a/src/engine/SCons/Defaults.xml +++ b/src/engine/SCons/Defaults.xml @@ -268,12 +268,8 @@ into a list of Dir instances relative to the target being built. The list of suffixes of files that will be scanned for imported D package files. -The default list is: +The default list is ['.d']. - - -['.d'] - @@ -584,9 +580,12 @@ in order to execute many of the global functions in this list from source code management systems. The default environment is a singleton, so the keyword arguments affect it only on the first call, on subsequent -calls the already-constructed object is returned. +calls the already-constructed object is returned and +any arguments are ignored. The default environment can be modified in the same way as any &consenv;. +Modifying the &defenv; has no effect on the environment +constructed by a subsequent &f-Environment; call. diff --git a/src/engine/SCons/Environment.xml b/src/engine/SCons/Environment.xml index a171f38..48980e2 100644 --- a/src/engine/SCons/Environment.xml +++ b/src/engine/SCons/Environment.xml @@ -1836,9 +1836,10 @@ Examples: -Program('foo', Glob('*.c')) -Zip('/tmp/everything', Glob('.??*') + Glob('*')) -sources = Glob('*.cpp', exclude=['os_*_specific_*.cpp']) + Glob('os_%s_specific_*.cpp'%currentOS) +Program("foo", Glob("*.c")) +Zip("/tmp/everything", Glob(".??*") + Glob("*")) +sources = Glob("*.cpp", exclude=["os_*_specific_*.cpp"]) + \ + Glob( "os_%s_specific_*.cpp" % currentOS) diff --git a/src/engine/SCons/Tool/applelink.xml b/src/engine/SCons/Tool/applelink.xml index fc0cf63..977f2ce 100644 --- a/src/engine/SCons/Tool/applelink.xml +++ b/src/engine/SCons/Tool/applelink.xml @@ -171,7 +171,7 @@ See its __doc__ string for a discussion of the format. - env.AppendUnique(FRAMEWORKS=Split('System Cocoa SystemConfiguration')) +env.AppendUnique(FRAMEWORKS=Split('System Cocoa SystemConfiguration')) @@ -213,7 +213,7 @@ See its __doc__ string for a discussion of the format. - env.AppendUnique(FRAMEWORKPATH='#myframeworkdir') +env.AppendUnique(FRAMEWORKPATH='#myframeworkdir') @@ -221,7 +221,7 @@ See its __doc__ string for a discussion of the format. - ... -Fmyframeworkdir +... -Fmyframeworkdir diff --git a/src/engine/SCons/Tool/jar.xml b/src/engine/SCons/Tool/jar.xml index 30c51f8..151dda1 100644 --- a/src/engine/SCons/Tool/jar.xml +++ b/src/engine/SCons/Tool/jar.xml @@ -120,7 +120,7 @@ If this is not set, then &cv-link-JARCOM; (the command line) is displayed. -env = Environment(JARCOMSTR = "JARchiving $SOURCES into $TARGET") +env = Environment(JARCOMSTR="JARchiving $SOURCES into $TARGET") diff --git a/src/engine/SCons/Tool/javac.xml b/src/engine/SCons/Tool/javac.xml index 893130b..b1548c7 100644 --- a/src/engine/SCons/Tool/javac.xml +++ b/src/engine/SCons/Tool/javac.xml @@ -174,7 +174,7 @@ env['ENV']['LANG'] = 'en_GB.UTF-8' -env = Environment(JAVACCOMSTR = "Compiling class files $TARGETS from $SOURCES") +env = Environment(JAVACCOMSTR="Compiling class files $TARGETS from $SOURCES") diff --git a/src/engine/SCons/Tool/javah.xml b/src/engine/SCons/Tool/javah.xml index 4d436b1..5a2840f 100644 --- a/src/engine/SCons/Tool/javah.xml +++ b/src/engine/SCons/Tool/javah.xml @@ -77,17 +77,18 @@ Examples: # builds java_native.h -classes = env.Java(target = 'classdir', source = 'src') -env.JavaH(target = 'java_native.h', source = classes) +classes = env.Java(target="classdir", source="src") +env.JavaH(target="java_native.h", source=classes) # builds include/package_foo.h and include/package_bar.h -env.JavaH(target = 'include', - source = ['package/foo.class', 'package/bar.class']) +env.JavaH(target="include", source=["package/foo.class", "package/bar.class"]) # builds export/foo.h and export/bar.h -env.JavaH(target = 'export', - source = ['classes/foo.class', 'classes/bar.class'], - JAVACLASSDIR = 'classes') +env.JavaH( + target="export", + source=["classes/foo.class", "classes/bar.class"], + JAVACLASSDIR="classes", +) @@ -120,7 +121,7 @@ If this is not set, then &cv-link-JAVAHCOM; (the command line) is displayed. -env = Environment(JAVAHCOMSTR = "Generating header/stub file(s) $TARGETS from $SOURCES") +env = Environment(JAVAHCOMSTR="Generating header/stub file(s) $TARGETS from $SOURCES") diff --git a/src/engine/SCons/Tool/msvs.xml b/src/engine/SCons/Tool/msvs.xml index b1f79b2..f6c9a39 100644 --- a/src/engine/SCons/Tool/msvs.xml +++ b/src/engine/SCons/Tool/msvs.xml @@ -422,7 +422,11 @@ env.MSVSProject(target='Bar' + env['MSVSPROJECTSUFFIX'], Example Usage: -env.MSVSSolution(target='Bar' + env['MSVSSOLUTIONSUFFIX'], projects=['bar' + env['MSVSPROJECTSUFFIX']], variant='Release') +env.MSVSSolution( + target="Bar" + env["MSVSSOLUTIONSUFFIX"], + projects=["bar" + env["MSVSPROJECTSUFFIX"]], + variant="Release", +) diff --git a/src/engine/SCons/Tool/packaging/__init__.xml b/src/engine/SCons/Tool/packaging/__init__.xml index 2b3c4fd..37e97ff 100644 --- a/src/engine/SCons/Tool/packaging/__init__.xml +++ b/src/engine/SCons/Tool/packaging/__init__.xml @@ -86,18 +86,19 @@ on a project that has packaging activated. -env = Environment(tools=['default', 'packaging']) -env.Install('/bin/', 'my_program') -env.Package( NAME = 'foo', - VERSION = '1.2.3', - PACKAGEVERSION = 0, - PACKAGETYPE = 'rpm', - LICENSE = 'gpl', - SUMMARY = 'balalalalal', - DESCRIPTION = 'this should be really really long', - X_RPM_GROUP = 'Application/fu', - SOURCE_URL = 'http://foo.org/foo-1.2.3.tar.gz' - ) +env = Environment(tools=["default", "packaging"]) +env.Install("/bin/", "my_program") +env.Package( + NAME="foo", + VERSION="1.2.3", + PACKAGEVERSION=0, + PACKAGETYPE="rpm", + LICENSE="gpl", + SUMMARY="balalalalal", + DESCRIPTION="this should be really really long", + X_RPM_GROUP="Application/fu", + SOURCE_URL="http://foo.org/foo-1.2.3.tar.gz", +) @@ -503,13 +504,14 @@ Added in version 3.1. env.Package( - NAME = 'foo', -... - X_RPM_EXTRADEFS = [ - '%define _unpackaged_files_terminate_build 0' - '%define _missing_doc_files_terminate_build 0' + NAME="foo", + ... + X_RPM_EXTRADEFS=[ + "%define _unpackaged_files_terminate_build 0" + "%define _missing_doc_files_terminate_build 0" ], -... ) + ... +) -- cgit v0.12 From 433c62479cc022ae7b07cf46a8c772cedae2842f Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Tue, 7 Apr 2020 10:04:24 -0600 Subject: [PR #3576] workaroud "oem" coding not on Py3.5 Use a Windows API to fetch the console output code page if Python < 3.6, where the encoding name "oem" is not yet defined (this was the original proposal in issue #3572) Signed-off-by: Mats Wichmann --- src/engine/SCons/Tool/MSCommon/common.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/engine/SCons/Tool/MSCommon/common.py b/src/engine/SCons/Tool/MSCommon/common.py index b90c25a..b2b5de9 100644 --- a/src/engine/SCons/Tool/MSCommon/common.py +++ b/src/engine/SCons/Tool/MSCommon/common.py @@ -249,14 +249,21 @@ def get_output(vcbat, args=None, env=None): # Ongoing problems getting non-corrupted text led to this # changing to "oem" from "mbcs" - the scripts run presumably # attached to a console, so some particular rules apply. + # Unfortunately, "oem" not defined in Python 3.5, so get another way + if sys.version_info.major == 3 and sys.version_info.minor < 6: + from ctypes import windll + + OEM = "cp{}".format(windll.kernel32.GetConsoleOutputCP()) + else: + OEM = "oem" if stderr: # TODO: find something better to do with stderr; # this at least prevents errors from getting swallowed. - sys.stderr.write(stderr.decode("oem")) + sys.stderr.write(stderr.decode(OEM)) if popen.wait() != 0: - raise IOError(stderr.decode("oem")) + raise IOError(stderr.decode(OEM)) - return stdout.decode("oem") + return stdout.decode(OEM) KEEPLIST = ( -- cgit v0.12 From f293a449ea7f9d036a85e4d880a2ba7da4751c6a Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 9 Mar 2020 11:49:40 -0700 Subject: Remove python27 code --- bin/SConsDoc.py | 68 ++++++++++++++++----------------------------------------- 1 file changed, 19 insertions(+), 49 deletions(-) diff --git a/bin/SConsDoc.py b/bin/SConsDoc.py index e560a44..1e88c5b 100644 --- a/bin/SConsDoc.py +++ b/bin/SConsDoc.py @@ -115,8 +115,6 @@ import re import sys import copy -PY2 = sys.version_info[0] == 2 - # Do we have libxml2/libxslt/lxml? has_libxml2 = True try: @@ -312,20 +310,14 @@ if not has_libxml2: @staticmethod def writeGenTree(root, fp): dt = DoctypeDeclaration() - try: - encfun = unicode # PY2 - except NameError: - encfun = str + encfun = str fp.write(etree.tostring(root, encoding=encfun, pretty_print=True, doctype=dt.createDoctype())) @staticmethod def writeTree(root, fpath): - try: - encfun = unicode # PY2 - except NameError: - encfun = "utf-8" + encfun = "utf-8" with open(fpath, 'wb') as fp: fp.write(etree.tostring(root, encoding=encfun, pretty_print=True)) @@ -859,45 +851,23 @@ class SConsDocHandler(object): # Parse it self.parseDomtree(t.root, t.xpath_context, t.nsmap) -# lifted from Ka-Ping Yee's way cool pydoc module. -if PY2: - def importfile(path): - """Import a Python source file or compiled file given its path.""" - import imp - magic = imp.get_magic() - with open(path, 'r') as ifp: - if ifp.read(len(magic)) == magic: - kind = imp.PY_COMPILED - else: - kind = imp.PY_SOURCE - filename = os.path.basename(path) - name, ext = os.path.splitext(filename) - with open(path, 'r') as ifp: - try: - module = imp.load_module(name, ifp, path, (ext, 'r', kind)) - except ImportError as e: - sys.stderr.write("Could not import %s: %s\n" % (path, e)) - return None - return module - -else: # PY3 version, from newer pydoc - def importfile(path): - """Import a Python source file or compiled file given its path.""" - from importlib.util import MAGIC_NUMBER - with open(path, 'rb') as ifp: - is_bytecode = MAGIC_NUMBER == ifp.read(len(MAGIC_NUMBER)) - filename = os.path.basename(path) - name, ext = os.path.splitext(filename) - if is_bytecode: - loader = importlib._bootstrap_external.SourcelessFileLoader(name, path) - else: - loader = importlib._bootstrap_external.SourceFileLoader(name, path) - # XXX We probably don't need to pass in the loader here. - spec = importlib.util.spec_from_file_location(name, path, loader=loader) - try: - return importlib._bootstrap._load(spec) - except ImportError: - raise ErrorDuringImport(path, sys.exc_info()) +def importfile(path): + """Import a Python source file or compiled file given its path.""" + from importlib.util import MAGIC_NUMBER + with open(path, 'rb') as ifp: + is_bytecode = MAGIC_NUMBER == ifp.read(len(MAGIC_NUMBER)) + filename = os.path.basename(path) + name, ext = os.path.splitext(filename) + if is_bytecode: + loader = importlib._bootstrap_external.SourcelessFileLoader(name, path) + else: + loader = importlib._bootstrap_external.SourceFileLoader(name, path) + # XXX We probably don't need to pass in the loader here. + spec = importlib.util.spec_from_file_location(name, path, loader=loader) + try: + return importlib._bootstrap._load(spec) + except ImportError: + raise ErrorDuringImport(path, sys.exc_info()) # Local Variables: # tab-width:4 -- cgit v0.12 From f1538fb9000a343d87676ce110076b7cc197cb0e Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 9 Mar 2020 22:29:14 -0400 Subject: add SKIP_DOC= variable to enable disabling buildign documents. Moved epydoc setup into site_scons/epydoc.py --- SConstruct | 550 ++++++++++++++++++++++-------------------------- doc/SConscript | 174 +++++---------- site_scons/epydoc.py | 100 +++++++++ site_scons/site_init.py | 3 +- 4 files changed, 408 insertions(+), 419 deletions(-) create mode 100644 site_scons/epydoc.py diff --git a/SConstruct b/SConstruct index 23797da..db71e0f 100644 --- a/SConstruct +++ b/SConstruct @@ -31,15 +31,10 @@ month_year = 'December 2019' # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # -import distutils.util -import distutils.command import fnmatch import os import os.path -import re -import stat import sys -import tempfile import time import socket import textwrap @@ -98,17 +93,24 @@ if git: git_status_lines = p.readlines() revision = ARGUMENTS.get('REVISION', '') + + def generate_build_id(revision): return revision + if not revision and git: with os.popen("%s rev-parse HEAD 2> /dev/null" % git, "r") as p: git_hash = p.read().strip() + + def generate_build_id(revision): result = git_hash if [l for l in git_status_lines if 'modified' in l]: result = result + '[MODIFIED]' return result + + revision = git_hash checkpoint = ARGUMENTS.get('CHECKPOINT', '') @@ -136,11 +138,10 @@ for a in addpaths: if a not in sys.path: sys.path.append(a) - # Re-exporting LD_LIBRARY_PATH is necessary if the Python version was # built with the --enable-shared option. -ENV = { 'PATH' : os.environ['PATH'] } +ENV = {'PATH': os.environ['PATH']} for key in ['LOGNAME', 'PYTHONPATH', 'LD_LIBRARY_PATH']: if key in os.environ: ENV[key] = os.environ[key] @@ -150,73 +151,74 @@ if not os.path.isabs(build_dir): build_dir = os.path.normpath(os.path.join(os.getcwd(), build_dir)) command_line_variables = [ - ("BUILDDIR=", "The directory in which to build the packages. " + - "The default is the './build' subdirectory."), - - ("BUILD_ID=", "An identifier for the specific build." + - "The default is the Subversion revision number."), - - ("BUILD_SYSTEM=", "The system on which the packages were built. " + - "The default is whatever hostname is returned " + - "by socket.gethostname(). If SOURCE_DATE_EPOCH " + - "env var is set, '_reproducible' is the default."), - - ("CHECKPOINT=", "The specific checkpoint release being packaged, " + - "which will be appended to the VERSION string. " + - "A value of CHECKPOINT=d will generate a string " + - "of 'd' plus today's date in the format YYYMMDD. " + - "A value of CHECKPOINT=r will generate a " + - "string of 'r' plus the Subversion revision " + - "number. Any other CHECKPOINT= string will be " + - "used as is. There is no default value."), - - ("DATE=", "The date string representing when the packaging " + - "build occurred. The default is the day and time " + - "the SConstruct file was invoked, in the format " + - "YYYY/MM/DD HH:MM:SS."), - - ("DEVELOPER=", "The developer who created the packages. " + - "The default is the first set environment " + - "variable from the list $USERNAME, $LOGNAME, $USER." + - "If the SOURCE_DATE_EPOCH env var is set, " + - "'_reproducible' is the default."), - - ("REVISION=", "The revision number of the source being built. " + - "The default is the git hash returned " + - "'git rev-parse HEAD', with an appended string of " + - "'[MODIFIED]' if there are any changes in the " + - "working copy."), - - ("VERSION=", "The SCons version being packaged. The default " + - "is the hard-coded value '%s' " % default_version + - "from this SConstruct file."), - + ("BUILDDIR=", "The directory in which to build the packages. " + + "The default is the './build' subdirectory."), + + ("BUILD_ID=", "An identifier for the specific build." + + "The default is the Subversion revision number."), + + ("BUILD_SYSTEM=", "The system on which the packages were built. " + + "The default is whatever hostname is returned " + + "by socket.gethostname(). If SOURCE_DATE_EPOCH " + + "env var is set, '_reproducible' is the default."), + + ("CHECKPOINT=", "The specific checkpoint release being packaged, " + + "which will be appended to the VERSION string. " + + "A value of CHECKPOINT=d will generate a string " + + "of 'd' plus today's date in the format YYYMMDD. " + + "A value of CHECKPOINT=r will generate a " + + "string of 'r' plus the Subversion revision " + + "number. Any other CHECKPOINT= string will be " + + "used as is. There is no default value."), + + ("DATE=", "The date string representing when the packaging " + + "build occurred. The default is the day and time " + + "the SConstruct file was invoked, in the format " + + "YYYY/MM/DD HH:MM:SS."), + + ("DEVELOPER=", "The developer who created the packages. " + + "The default is the first set environment " + + "variable from the list $USERNAME, $LOGNAME, $USER." + + "If the SOURCE_DATE_EPOCH env var is set, " + + "'_reproducible' is the default."), + + ("REVISION=", "The revision number of the source being built. " + + "The default is the git hash returned " + + "'git rev-parse HEAD', with an appended string of " + + "'[MODIFIED]' if there are any changes in the " + + "working copy."), + + ("VERSION=", "The SCons version being packaged. The default " + + "is the hard-coded value '%s' " % default_version + + "from this SConstruct file."), + + ("SKIP_DOC=","Skip building all documents. The default is False (build docs)"), ] Default('.', build_dir) packaging_flavors = [ - ('tar-gz', "The normal .tar.gz file for end-user installation."), - ('local-tar-gz', "A .tar.gz file for dropping into other software " + - "for local use."), - ('zip', "The normal .zip file for end-user installation."), - ('local-zip', "A .zip file for dropping into other software " + - "for local use."), - ('src-tar-gz', "A .tar.gz file containing all the source " + - "(including tests and documentation)."), - ('src-zip', "A .zip file containing all the source " + - "(including tests and documentation)."), + ('tar-gz', "The normal .tar.gz file for end-user installation."), + ('local-tar-gz', "A .tar.gz file for dropping into other software " + + "for local use."), + ('zip', "The normal .zip file for end-user installation."), + ('local-zip', "A .zip file for dropping into other software " + + "for local use."), + ('src-tar-gz', "A .tar.gz file containing all the source " + + "(including tests and documentation)."), + ('src-zip', "A .zip file containing all the source " + + "(including tests and documentation)."), ] -test_tar_gz_dir = os.path.join(build_dir, "test-tar-gz") -test_src_tar_gz_dir = os.path.join(build_dir, "test-src-tar-gz") +test_tar_gz_dir = os.path.join(build_dir, "test-tar-gz") +test_src_tar_gz_dir = os.path.join(build_dir, "test-src-tar-gz") test_local_tar_gz_dir = os.path.join(build_dir, "test-local-tar-gz") -test_zip_dir = os.path.join(build_dir, "test-zip") -test_src_zip_dir = os.path.join(build_dir, "test-src-zip") -test_local_zip_dir = os.path.join(build_dir, "test-local-zip") +test_zip_dir = os.path.join(build_dir, "test-zip") +test_src_zip_dir = os.path.join(build_dir, "test-src-zip") +test_local_zip_dir = os.path.join(build_dir, "test-local-zip") -unpack_tar_gz_dir = os.path.join(build_dir, "unpack-tar-gz") -unpack_zip_dir = os.path.join(build_dir, "unpack-zip") +unpack_tar_gz_dir = os.path.join(build_dir, "unpack-tar-gz") +unpack_zip_dir = os.path.join(build_dir, "unpack-zip") if is_windows(): tar_hflag = '' @@ -227,9 +229,6 @@ else: python_project_subinst_dir = os.path.join("lib", project) project_script_subinst_dir = 'bin' - - - indent_fmt = ' %-26s ' Help("""\ @@ -243,9 +242,9 @@ aliases = sorted(packaging_flavors + [('doc', 'The SCons documentation.')]) for alias, help_text in aliases: tw = textwrap.TextWrapper( - width = 78, - initial_indent = indent_fmt % alias, - subsequent_indent = indent_fmt % '' + ' ', + width=78, + initial_indent=indent_fmt % alias, + subsequent_indent=indent_fmt % '' + ' ', ) Help(tw.fill(help_text) + '\n') @@ -256,61 +255,57 @@ The following command-line variables can be set: for variable, help_text in command_line_variables: tw = textwrap.TextWrapper( - width = 78, - initial_indent = indent_fmt % variable, - subsequent_indent = indent_fmt % '' + ' ', + width=78, + initial_indent=indent_fmt % variable, + subsequent_indent=indent_fmt % '' + ' ', ) Help(tw.fill(help_text) + '\n') - - - revaction = SCons_revision -revbuilder = Builder(action = Action(SCons_revision, - varlist=['COPYRIGHT', 'VERSION'])) - +revbuilder = Builder(action=Action(SCons_revision, + varlist=['COPYRIGHT', 'VERSION'])) # Just make copies, don't symlink them. SetOption('duplicate', 'copy') env = Environment( - ENV = ENV, + ENV=ENV, - BUILD = build_id, - BUILDDIR = build_dir, - BUILDSYS = build_system, - COPYRIGHT = copyright, - DATE = date, - DEB_DATE = deb_date, - DEVELOPER = developer, - DISTDIR = os.path.join(build_dir, 'dist'), - MONTH_YEAR = month_year, - REVISION = revision, - VERSION = version, + BUILD=build_id, + BUILDDIR=build_dir, + BUILDSYS=build_system, + COPYRIGHT=copyright, + DATE=date, + DEB_DATE=deb_date, + DEVELOPER=developer, + DISTDIR=os.path.join(build_dir, 'dist'), + MONTH_YEAR=month_year, + REVISION=revision, + VERSION=version, - TAR_HFLAG = tar_hflag, + TAR_HFLAG=tar_hflag, - ZIP = zip, - ZIPFLAGS = '-r', - UNZIP = unzip, - UNZIPFLAGS = '-o -d $UNPACK_ZIP_DIR', + ZIP=zip, + ZIPFLAGS='-r', + UNZIP=unzip, + UNZIPFLAGS='-o -d $UNPACK_ZIP_DIR', - ZCAT = zcat, + ZCAT=zcat, - TEST_SRC_TAR_GZ_DIR = test_src_tar_gz_dir, - TEST_SRC_ZIP_DIR = test_src_zip_dir, - TEST_TAR_GZ_DIR = test_tar_gz_dir, - TEST_ZIP_DIR = test_zip_dir, + TEST_SRC_TAR_GZ_DIR=test_src_tar_gz_dir, + TEST_SRC_ZIP_DIR=test_src_zip_dir, + TEST_TAR_GZ_DIR=test_tar_gz_dir, + TEST_ZIP_DIR=test_zip_dir, - UNPACK_TAR_GZ_DIR = unpack_tar_gz_dir, - UNPACK_ZIP_DIR = unpack_zip_dir, + UNPACK_TAR_GZ_DIR=unpack_tar_gz_dir, + UNPACK_ZIP_DIR=unpack_zip_dir, - BUILDERS = { 'SCons_revision' : revbuilder, - 'SOElim' : soelimbuilder }, + BUILDERS={'SCons_revision': revbuilder, + 'SOElim': soelimbuilder}, - PYTHON = '"%s"' % sys.executable, - PYTHONFLAGS = '-tt', - ) + PYTHON='"%s"' % sys.executable, + PYTHONFLAGS='-tt', +) Version_values = [Value(version), Value(build_id)] @@ -332,128 +327,90 @@ Version_values = [Value(version), Value(build_id)] from distutils.sysconfig import get_python_lib - python_scons = { - 'pkg' : 'python-' + project, - 'src_subdir' : 'engine', - 'inst_subdir' : get_python_lib(), - - 'debian_deps' : [ - 'debian/changelog', - 'debian/compat', - 'debian/control', - 'debian/copyright', - 'debian/dirs', - 'debian/docs', - 'debian/postinst', - 'debian/prerm', - 'debian/rules', - ], - - 'files' : [ 'LICENSE.txt', - 'README.txt', - 'setup.cfg', - 'setup.py', - ], - - 'filemap' : { - 'LICENSE.txt' : '../LICENSE.txt' - }, - - 'buildermap' : {}, - - 'explicit_deps' : { - 'SCons/__init__.py' : Version_values, - }, -} + 'pkg': 'python-' + project, + 'src_subdir': 'engine', + 'inst_subdir': get_python_lib(), + + 'files': ['LICENSE.txt', + 'README.txt', + 'setup.cfg', + 'setup.py', + ], + 'filemap': { + 'LICENSE.txt': '../LICENSE.txt' + }, + + 'buildermap': {}, + + 'explicit_deps': { + 'SCons/__init__.py': Version_values, + }, +} scons_script = { - 'pkg' : project + '-script', - 'src_subdir' : 'script', - 'inst_subdir' : 'bin', - - 'debian_deps' : [ - 'debian/changelog', - 'debian/compat', - 'debian/control', - 'debian/copyright', - 'debian/dirs', - 'debian/docs', - 'debian/postinst', - 'debian/prerm', - 'debian/rules', - ], - - 'files' : [ - 'LICENSE.txt', - 'README.txt', - 'setup.cfg', - 'setup.py', - ], - - 'filemap' : { - 'LICENSE.txt' : '../LICENSE.txt', - 'scons' : 'scons.py', - 'sconsign' : 'sconsign.py', - 'scons-time' : 'scons-time.py', - 'scons-configure-cache' : 'scons-configure-cache.py', - }, - - 'buildermap' : {}, - - 'explicit_deps' : { - 'scons' : Version_values, - 'sconsign' : Version_values, - }, + 'pkg': project + '-script', + 'src_subdir': 'script', + 'inst_subdir': 'bin', + + 'files': [ + 'LICENSE.txt', + 'README.txt', + 'setup.cfg', + 'setup.py', + ], + + 'filemap': { + 'LICENSE.txt': '../LICENSE.txt', + 'scons': 'scons.py', + 'sconsign': 'sconsign.py', + 'scons-time': 'scons-time.py', + 'scons-configure-cache': 'scons-configure-cache.py', + }, + + 'buildermap': {}, + + 'explicit_deps': { + 'scons': Version_values, + 'sconsign': Version_values, + }, } scons = { - 'pkg' : project, - - 'debian_deps' : [ - 'debian/changelog', - 'debian/compat', - 'debian/control', - 'debian/copyright', - 'debian/dirs', - 'debian/docs', - 'debian/postinst', - 'debian/prerm', - 'debian/rules', - ], - - 'files' : [ - 'CHANGES.txt', - 'LICENSE.txt', - 'README.txt', - 'RELEASE.txt', - 'scons.1', - 'sconsign.1', - 'scons-time.1', - 'script/scons.bat', - 'setup.cfg', - 'setup.py', - ], - - 'filemap' : { - 'scons.1' : '$BUILDDIR/doc/man/scons.1', - 'sconsign.1' : '$BUILDDIR/doc/man/sconsign.1', - 'scons-time.1' : '$BUILDDIR/doc/man/scons-time.1', - }, - - 'buildermap' : { - 'scons.1' : env.SOElim, - 'sconsign.1' : env.SOElim, - 'scons-time.1' : env.SOElim, - }, - - 'subpkgs' : [ python_scons, scons_script ], - - 'subinst_dirs' : { - 'python-' + project : python_project_subinst_dir, - project + '-script' : project_script_subinst_dir, - }, + 'pkg': project, + + 'files': [ + 'CHANGES.txt', + 'LICENSE.txt', + 'README.txt', + 'RELEASE.txt', + 'scons.1', + 'sconsign.1', + 'scons-time.1', + 'script/scons.bat', + 'setup.cfg', + 'setup.py', + ], + + 'filemap': { + 'scons.1': '$BUILDDIR/doc/man/scons.1', + 'sconsign.1': '$BUILDDIR/doc/man/sconsign.1', + 'scons-time.1': '$BUILDDIR/doc/man/scons-time.1', + }, + + 'buildermap': { + 'scons.1': env.SOElim, + 'sconsign.1': env.SOElim, + 'scons-time.1': env.SOElim, + }, + + 'subpkgs': [python_scons, scons_script], + + 'subinst_dirs': { + 'python-' + project: python_project_subinst_dir, + project + '-script': project_script_subinst_dir, + }, } scripts = ['scons', 'sconsign', 'scons-time', 'scons-configure-cache'] @@ -461,7 +418,7 @@ scripts = ['scons', 'sconsign', 'scons-time', 'scons-configure-cache'] src_deps = [] src_files = [] -for p in [ scons ]: +for p in [scons]: # # Initialize variables with the right directories for this package. # @@ -483,7 +440,6 @@ for p in [ scons ]: 'dist', "%s.%s.zip" % (pkg_version, platform)) - # # Update the environment with the relevant information # for this package. @@ -493,9 +449,9 @@ for p in [ scons ]: # to the directory in which setup.py exists. # setup_py = os.path.join(build, 'setup.py') - env.Replace(PKG = pkg, - PKG_VERSION = pkg_version, - SETUP_PY = '"%s"' % setup_py) + env.Replace(PKG=pkg, + PKG_VERSION=pkg_version, + SETUP_PY='"%s"' % setup_py) Local(setup_py) # @@ -511,7 +467,6 @@ for p in [ scons ]: MANIFEST_in_list = [] - if 'subpkgs' in p: # # This package includes some sub-packages. Read up their @@ -571,6 +526,7 @@ for p in [ scons ]: src_files.append("MANIFEST") MANIFEST_in_list.append(os.path.join(src, 'MANIFEST.in')) + def write_src_files(target, source, **kw): global src_files src_files.sort() @@ -578,6 +534,8 @@ for p in [ scons ]: for file in src_files: f.write(file + "\n") return 0 + + env.Command(os.path.join(build, 'MANIFEST'), MANIFEST_in_list, write_src_files) @@ -605,10 +563,10 @@ for p in [ scons ]: src_deps.append(tar_gz) - distutils_targets.extend([ tar_gz, platform_tar_gz ]) + distutils_targets.extend([tar_gz, platform_tar_gz]) - dist_tar_gz = env.Install('$DISTDIR', tar_gz) - dist_platform_tar_gz = env.Install('$DISTDIR', platform_tar_gz) + dist_tar_gz = env.Install('$DISTDIR', tar_gz) + dist_platform_tar_gz = env.Install('$DISTDIR', platform_tar_gz) Local(dist_tar_gz, dist_platform_tar_gz) AddPostAction(dist_tar_gz, Chmod(dist_tar_gz, 0o644)) AddPostAction(dist_platform_tar_gz, Chmod(dist_platform_tar_gz, 0o644)) @@ -627,10 +585,10 @@ for p in [ scons ]: unpack_tar_gz_files = [os.path.join(unpack_tar_gz_dir, pkg_version, x) for x in src_files] env.Command(unpack_tar_gz_files, dist_tar_gz, [ - Delete(os.path.join(unpack_tar_gz_dir, pkg_version)), - "$ZCAT $SOURCES > .temp", - "tar xf .temp -C $UNPACK_TAR_GZ_DIR", - Delete(".temp"), + Delete(os.path.join(unpack_tar_gz_dir, pkg_version)), + "$ZCAT $SOURCES > .temp", + "tar xf .temp -C $UNPACK_TAR_GZ_DIR", + Delete(".temp"), ]) # @@ -651,7 +609,7 @@ for p in [ scons ]: Delete(os.path.join(unpack_tar_gz_dir, pkg_version, 'build')), Delete("$TEST_TAR_GZ_DIR"), '$PYTHON $PYTHONFLAGS "%s" install "--prefix=$TEST_TAR_GZ_DIR" --standalone-lib' % \ - os.path.join(unpack_tar_gz_dir, pkg_version, 'setup.py'), + os.path.join(unpack_tar_gz_dir, pkg_version, 'setup.py'), ]) # @@ -662,10 +620,11 @@ for p in [ scons ]: digest = os.path.join(gentoo, 'files', 'digest-scons-%s' % version) env.Command(ebuild, os.path.join('gentoo', 'scons.ebuild.in'), SCons_revision) + def Digestify(target, source, env): import hashlib src = source[0].rfile() - with open(str(src),'rb') as f: + with open(str(src), 'rb') as f: contents = f.read() m = hashlib.md5() m.update(contents) @@ -673,6 +632,8 @@ for p in [ scons ]: bytes = os.stat(str(src))[6] with open(str(target[0]), 'w') as f: f.write("MD5 %s %s %d\n" % (sig, src.name, bytes)) + + env.Command(digest, tar_gz, Digestify) if not zipit: @@ -683,10 +644,10 @@ for p in [ scons ]: src_deps.append(zip) - distutils_targets.extend([ zip, platform_zip ]) + distutils_targets.extend([zip, platform_zip]) - dist_zip = env.Install('$DISTDIR', zip) - dist_platform_zip = env.Install('$DISTDIR', platform_zip) + dist_zip = env.Install('$DISTDIR', zip) + dist_platform_zip = env.Install('$DISTDIR', platform_zip) Local(dist_zip, dist_platform_zip) AddPostAction(dist_zip, Chmod(dist_zip, 0o644)) AddPostAction(dist_platform_zip, Chmod(dist_platform_zip, 0o644)) @@ -696,7 +657,7 @@ for p in [ scons ]: # build/unpack-zip/scons-{version}. # unpack_zip_files = [os.path.join(unpack_zip_dir, pkg_version, x) - for x in src_files] + for x in src_files] env.Command(unpack_zip_files, dist_zip, [ Delete(os.path.join(unpack_zip_dir, pkg_version)), @@ -721,11 +682,9 @@ for p in [ scons ]: Delete(os.path.join(unpack_zip_dir, pkg_version, 'build')), Delete("$TEST_ZIP_DIR"), '$PYTHON $PYTHONFLAGS "%s" install "--prefix=$TEST_ZIP_DIR" --standalone-lib' % \ - os.path.join(unpack_zip_dir, pkg_version, 'setup.py'), + os.path.join(unpack_zip_dir, pkg_version, 'setup.py'), ]) - - # # Use the Python distutils to generate the appropriate packages. # @@ -742,8 +701,8 @@ for p in [ scons ]: for format in distutils_formats: commands.append("$PYTHON $PYTHONFLAGS $SETUP_PY bdist_dumb -f %s" % format) - commands.append("$PYTHON $PYTHONFLAGS $SETUP_PY sdist --formats=%s" % \ - ','.join(distutils_formats)) + commands.append("$PYTHON $PYTHONFLAGS $SETUP_PY sdist --formats=%s" % \ + ','.join(distutils_formats)) env.Command(distutils_targets, build_src_files, commands) @@ -766,7 +725,7 @@ for p in [ scons ]: commands = [ Delete(build_dir_local), '$PYTHON $PYTHONFLAGS $SETUP_PY install "--install-script=%s" "--install-lib=%s" --no-install-man --no-compile --standalone-lib --no-version-script' % \ - (build_dir_local, build_dir_local_slv), + (build_dir_local, build_dir_local_slv), ] for script in scripts: @@ -813,7 +772,7 @@ for p in [ scons ]: if zipit: env.Command(dist_local_zip, local_targets, zipit, - CD = build_dir_local, PSV = '.') + CD=build_dir_local, PSV='.') unpack_targets = [os.path.join(test_local_zip_dir, x) for x in rf] commands = [Delete(test_local_zip_dir), @@ -821,7 +780,7 @@ for p in [ scons ]: unzipit] env.Command(unpack_targets, dist_local_zip, unzipit, - UNPACK_ZIP_DIR = test_local_zip_dir) + UNPACK_ZIP_DIR=test_local_zip_dir) # # @@ -839,7 +798,6 @@ files = [ 'runtest.py', ] - # # Documentation. # @@ -859,7 +817,7 @@ if git_status_lines: # sfiles = [l.split()[-1] for l in slines] pass else: - print("Not building in a Git tree; skipping building src package.") + print("Not building in a Git tree; skipping building src package.") if sfiles: remove_patterns = [ @@ -886,7 +844,7 @@ if sfiles: for file in sfiles: if file.endswith('jpg') or file.endswith('png'): # don't revision binary files. - env.Install(os.path.dirname(os.path.join(b_ps,file)), file) + env.Install(os.path.dirname(os.path.join(b_ps, file)), file) else: env.SCons_revision(os.path.join(b_ps, file), file) @@ -902,7 +860,6 @@ if sfiles: Local(*b_ps_files) if gzip: - env.Command(src_tar_gz, b_psv_stamp, "tar cz${TAR_HFLAG} -f $TARGET -C build %s" % psv) @@ -945,29 +902,28 @@ if sfiles: ENV['SCONS_LIB_DIR'] = scons_lib_dir ENV['USERNAME'] = developer env.Command(dfiles, unpack_tar_gz_files, - [ - Delete(os.path.join(unpack_tar_gz_dir, - psv, - 'build', - 'scons', - 'build')), - Delete("$TEST_SRC_TAR_GZ_DIR"), - 'cd "%s" && $PYTHON $PYTHONFLAGS "%s" "%s" VERSION="$VERSION"' % \ - (os.path.join(unpack_tar_gz_dir, psv), - os.path.join('src', 'script', 'scons.py'), - os.path.join('build', 'scons')), - '$PYTHON $PYTHONFLAGS "%s" install "--prefix=$TEST_SRC_TAR_GZ_DIR" --standalone-lib' % \ - os.path.join(unpack_tar_gz_dir, - psv, - 'build', - 'scons', - 'setup.py'), - ], - ENV = ENV) + [ + Delete(os.path.join(unpack_tar_gz_dir, + psv, + 'build', + 'scons', + 'build')), + Delete("$TEST_SRC_TAR_GZ_DIR"), + 'cd "%s" && $PYTHON $PYTHONFLAGS "%s" "%s" VERSION="$VERSION"' % \ + (os.path.join(unpack_tar_gz_dir, psv), + os.path.join('src', 'script', 'scons.py'), + os.path.join('build', 'scons')), + '$PYTHON $PYTHONFLAGS "%s" install "--prefix=$TEST_SRC_TAR_GZ_DIR" --standalone-lib' % \ + os.path.join(unpack_tar_gz_dir, + psv, + 'build', + 'scons', + 'setup.py'), + ], + ENV=ENV) if zipit: - - env.Command(src_zip, b_psv_stamp, zipit, CD = 'build', PSV = psv) + env.Command(src_zip, b_psv_stamp, zipit, CD='build', PSV=psv) # # Unpack the archive into build/unpack/scons-{version}. @@ -999,31 +955,29 @@ if sfiles: ENV['SCONS_LIB_DIR'] = scons_lib_dir ENV['USERNAME'] = developer env.Command(dfiles, unpack_zip_files, - [ - Delete(os.path.join(unpack_zip_dir, - psv, - 'build', - 'scons', - 'build')), - Delete("$TEST_SRC_ZIP_DIR"), - 'cd "%s" && $PYTHON $PYTHONFLAGS "%s" "%s" VERSION="$VERSION"' % \ - (os.path.join(unpack_zip_dir, psv), - os.path.join('src', 'script', 'scons.py'), - os.path.join('build', 'scons')), - '$PYTHON $PYTHONFLAGS "%s" install "--prefix=$TEST_SRC_ZIP_DIR" --standalone-lib' % \ - os.path.join(unpack_zip_dir, - psv, - 'build', - 'scons', - 'setup.py'), - ], - ENV = ENV) + [ + Delete(os.path.join(unpack_zip_dir, + psv, + 'build', + 'scons', + 'build')), + Delete("$TEST_SRC_ZIP_DIR"), + 'cd "%s" && $PYTHON $PYTHONFLAGS "%s" "%s" VERSION="$VERSION"' % \ + (os.path.join(unpack_zip_dir, psv), + os.path.join('src', 'script', 'scons.py'), + os.path.join('build', 'scons')), + '$PYTHON $PYTHONFLAGS "%s" install "--prefix=$TEST_SRC_ZIP_DIR" --standalone-lib' % \ + os.path.join(unpack_zip_dir, + psv, + 'build', + 'scons', + 'setup.py'), + ], + ENV=ENV) for pf, help_text in packaging_flavors: Alias(pf, [ - os.path.join(build_dir, 'test-'+pf), + os.path.join(build_dir, 'test-' + pf), os.path.join(build_dir, 'testing/framework'), os.path.join(build_dir, 'runtest.py'), ]) - - diff --git a/doc/SConscript b/doc/SConscript index 8f6d1cd..580f4ca 100644 --- a/doc/SConscript +++ b/doc/SConscript @@ -54,6 +54,11 @@ if not fop and not xep: print("doc: No PDF renderer found (fop|xep)!") skip_doc = True + +skip_doc_arg = ARGUMENTS.get('SKIP_DOC') +if skip_doc_arg is not None: + skip_doc = skip_doc_arg in ['True', '1', 'true'] + # # --- Configure build # @@ -62,7 +67,6 @@ env = env.Clone() build = os.path.join(build_dir, 'doc') -epydoc_cli = whereis('epydoc') gs = whereis('gs') lynx = whereis('lynx') @@ -494,127 +498,57 @@ else: tar_list.extend([css_file]) Local(css_file) -if not epydoc_cli: - try: - import epydoc - except ImportError: - epydoc = None +if not skip_doc: + if not epydoc_cli and not epydoc: + print("doc: epydoc not found, skipping building API documentation.") else: - # adding Epydoc builder using imported module - def epydoc_builder_action(target, source, env): - """ - Take a list of `source` files and build docs for them in - `target` dir. - - `target` and `source` are lists. - - Uses OUTDIR and EPYDOCFLAGS environment variables. - - http://www.scons.org/doc/2.0.1/HTML/scons-user/x3594.html - """ - - # the epydoc build process is the following: - # 1. build documentation index - # 2. feed doc index to writer for docs - - from epydoc.docbuilder import build_doc_index - from epydoc.docwriter.html import HTMLWriter - from epydoc.docwriter.latex import LatexWriter - - # first arg is a list where can be names of python package dirs, - # python files, object names or objects itself - docindex = build_doc_index([str(src) for src in source]) - if docindex is None: - return -1 - - if env['EPYDOCFLAGS'] == '--html': - html_writer = HTMLWriter(docindex, - docformat='restructuredText', - prj_name='SCons', - prj_url='http://www.scons.org/') - try: - html_writer.write(env['OUTDIR']) - except OSError: # If directory cannot be created or any file cannot - # be created or written to. - return -2 - - """ - # PDF support requires external Linux utilites, so it's not crossplatform. - # Leaving for now. - # http://epydoc.svn.sourceforge.net/viewvc/epydoc/trunk/epydoc/src/epydoc/cli.py - - elif env['EPYDOCFLAGS'] == '--pdf': - pdf_writer = LatexWriter(docindex, - docformat='restructuredText', - prj_name='SCons', - prj_url='http://www.scons.org/') - """ - return 0 - - epydoc_commands = [ - Delete('$OUTDIR'), - epydoc_builder_action, - Touch('$TARGET'), - ] - -else: # epydoc_cli is found - epydoc_commands = [ - Delete('$OUTDIR'), - '$EPYDOC $EPYDOCFLAGS --debug --output $OUTDIR --docformat=restructuredText --name SCons --url http://www.scons.org/ $SOURCES', - Touch('$TARGET'), - ] - - -if not epydoc_cli and not epydoc: - print("doc: epydoc not found, skipping building API documentation.") -else: - # XXX Should be in common with reading the same thing in - # the SConstruct file. - # bootstrap.py runs outside of SCons, so need to process the path - e = Dir(os.path.join('#src', 'engine')).rstr() - sources = bootstrap.parseManifestLines(e, os.path.join(e, 'MANIFEST.in')) + # XXX Should be in common with reading the same thing in + # the SConstruct file. + # bootstrap.py runs outside of SCons, so need to process the path + e = Dir(os.path.join('#src', 'engine')).rstr() + sources = bootstrap.parseManifestLines(e, os.path.join(e, 'MANIFEST.in')) - # Omit some files: - # - # Don't omit Platform as we need Platform.virtualenv for the examples to be run - # sources = [x for x in sources if x.find('Platform') == -1] - sources = [x for x in sources if x.find('Tool') == -1] - sources = [x for x in sources if x.find('Options') == -1] - - e = os.path.join(build, '..', 'scons', 'engine') - sources = [os.path.join(e, x) for x in sources] - - htmldir = os.path.join(build, 'HTML', 'scons-api') - env.Command('${OUTDIR}/index.html', sources, epydoc_commands, - EPYDOC=epydoc_cli, EPYDOCFLAGS='--html', OUTDIR=htmldir) - tar_deps.append(htmldir) - tar_list.append(htmldir) - - if sys.platform == 'darwin' or not epydoc_cli: - print("doc: command line epydoc is not found, skipping PDF/PS/Tex output") - else: - # PDF and PostScript and TeX are built from the - # same invocation. - api_dir = os.path.join(build, 'scons-api') - api_pdf = os.path.join(api_dir, 'api.pdf') - api_ps = os.path.join(api_dir, 'api.ps') - api_tex = os.path.join(api_dir, 'api.tex') - api_targets = [api_pdf, api_ps, api_tex] - env.Command(api_targets, sources, epydoc_commands, - EPYDOC=epydoc_cli, EPYDOCFLAGS='--pdf', OUTDIR=api_dir) - Local(api_targets) - - pdf_install = os.path.join(build, 'PDF', 'scons-api.pdf') - env.InstallAs(pdf_install, api_pdf) - tar_deps.append(pdf_install) - tar_list.append(pdf_install) - Local(pdf_install) - - ps_install = os.path.join(build, 'PS', 'scons-api.ps') - env.InstallAs(ps_install, api_ps) - tar_deps.append(ps_install) - tar_list.append(ps_install) - Local(ps_install) + # Omit some files: + # + # Don't omit Platform as we need Platform.virtualenv for the examples to be run + # sources = [x for x in sources if x.find('Platform') == -1] + sources = [x for x in sources if x.find('Tool') == -1] + sources = [x for x in sources if x.find('Options') == -1] + + e = os.path.join(build, '..', 'scons', 'engine') + sources = [os.path.join(e, x) for x in sources] + + htmldir = os.path.join(build, 'HTML', 'scons-api') + env.Command('${OUTDIR}/index.html', sources, epydoc_commands, + EPYDOC=epydoc_cli, EPYDOCFLAGS='--html', OUTDIR=htmldir) + tar_deps.append(htmldir) + tar_list.append(htmldir) + + if sys.platform == 'darwin' or not epydoc_cli: + print("doc: command line epydoc is not found, skipping PDF/PS/Tex output") + else: + # PDF and PostScript and TeX are built from the + # same invocation. + api_dir = os.path.join(build, 'scons-api') + api_pdf = os.path.join(api_dir, 'api.pdf') + api_ps = os.path.join(api_dir, 'api.ps') + api_tex = os.path.join(api_dir, 'api.tex') + api_targets = [api_pdf, api_ps, api_tex] + env.Command(api_targets, sources, epydoc_commands, + EPYDOC=epydoc_cli, EPYDOCFLAGS='--pdf', OUTDIR=api_dir) + Local(api_targets) + + pdf_install = os.path.join(build, 'PDF', 'scons-api.pdf') + env.InstallAs(pdf_install, api_pdf) + tar_deps.append(pdf_install) + tar_list.append(pdf_install) + Local(pdf_install) + + ps_install = os.path.join(build, 'PS', 'scons-api.ps') + env.InstallAs(ps_install, api_ps) + tar_deps.append(ps_install) + tar_list.append(ps_install) + Local(ps_install) # # Now actually create the tar file of the documentation, diff --git a/site_scons/epydoc.py b/site_scons/epydoc.py new file mode 100644 index 0000000..149e9dc --- /dev/null +++ b/site_scons/epydoc.py @@ -0,0 +1,100 @@ +# +# Setup epydoc builder +# + +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# from Utilities import whereis +from SCons.Script import Delete, Touch, WhereIs + +epydoc_cli = WhereIs('epydoc') + + +if not epydoc_cli: + try: + import epydoc + except ImportError: + epydoc = None + else: + # adding Epydoc builder using imported module + def epydoc_builder_action(target, source, env): + """ + Take a list of `source` files and build docs for them in + `target` dir. + + `target` and `source` are lists. + + Uses OUTDIR and EPYDOCFLAGS environment variables. + + http://www.scons.org/doc/2.0.1/HTML/scons-user/x3594.html + """ + + # the epydoc build process is the following: + # 1. build documentation index + # 2. feed doc index to writer for docs + + from epydoc.docbuilder import build_doc_index + from epydoc.docwriter.html import HTMLWriter + from epydoc.docwriter.latex import LatexWriter + + # first arg is a list where can be names of python package dirs, + # python files, object names or objects itself + docindex = build_doc_index([str(src) for src in source]) + if docindex is None: + return -1 + + if env['EPYDOCFLAGS'] == '--html': + html_writer = HTMLWriter(docindex, + docformat='restructuredText', + prj_name='SCons', + prj_url='http://www.scons.org/') + try: + html_writer.write(env['OUTDIR']) + except OSError: # If directory cannot be created or any file cannot + # be created or written to. + return -2 + + """ + # PDF support requires external Linux utilites, so it's not crossplatform. + # Leaving for now. + # http://epydoc.svn.sourceforge.net/viewvc/epydoc/trunk/epydoc/src/epydoc/cli.py + + elif env['EPYDOCFLAGS'] == '--pdf': + pdf_writer = LatexWriter(docindex, + docformat='restructuredText', + prj_name='SCons', + prj_url='http://www.scons.org/') + """ + return 0 + + epydoc_commands = [ + Delete('$OUTDIR'), + epydoc_builder_action, + Touch('$TARGET'), + ] + +else: # epydoc_cli is found + epydoc_commands = [ + Delete('$OUTDIR'), + '$EPYDOC $EPYDOCFLAGS --debug --output $OUTDIR --docformat=restructuredText --name SCons --url http://www.scons.org/ $SOURCES', + Touch('$TARGET'), + ] diff --git a/site_scons/site_init.py b/site_scons/site_init.py index b62eb37..d4df473 100644 --- a/site_scons/site_init.py +++ b/site_scons/site_init.py @@ -1,4 +1,5 @@ from SConsRevision import SCons_revision from Utilities import is_windows, whereis, platform, deb_date from zip_utils import unzipit, zipit, zcat -from soe_utils import soelim, soscan, soelimbuilder \ No newline at end of file +from soe_utils import soelim, soscan, soelimbuilder +from epydoc import epydoc_cli, epydoc_commands \ No newline at end of file -- cgit v0.12 From 7c466f714b9892d953ff2b91c7b0214180417000 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sun, 15 Mar 2020 21:20:38 -0700 Subject: Initial checking of modernized setup.py. TODO: entry_points for every other script (scons.py is done) --- MANIFEST.in | 3 +++ setup.cfg | 64 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ setup.py | 6 ++++++ 3 files changed, 73 insertions(+) create mode 100644 MANIFEST.in create mode 100644 setup.cfg create mode 100644 setup.py diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..4e4ae5a --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,3 @@ +recursive-exclude src/engine *Tests.py +global-exclude *Tests.py +recursive-include src/engine/SCons/Tool/docbook * diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..d44c1f5 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,64 @@ +[metadata] +name = SCons +version=3.9.9a99 +license = MIT +author = William Deegan +author_email =bill@baddogconsulting.com +long_description = file: README.rst +description = Open Source next-generation build tool. +group = Development/Tools +license_file = src/LICENSE.txt + + +url = http://www.scons.org/ +project-urls = + Documentation = https://scons.org/documentation.html + Twitter = https://twitter.com/SConsProject + GitHub = https://github.com/SCons/scons + Bug-Tracker = https://github.com/SCons/scons/issues + + +classifiers = + Development Status :: 5 - Production/Stable + Topic :: Utilities + Programming Language :: Python + Programming Language :: Python :: 3 + Programming Language :: Python :: 3 :: Only + Programming Language :: Python :: 3.5 + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Environment :: Console + Intended Audience :: Developers + License :: OSI Approved :: MIT License + Operating System :: POSIX :: Linux + Operating System :: Unix + Operating System :: MacOS + Operating System :: Microsoft :: Windows + + +[options] +zip_safe = False +python_requires = >=3.5 +install_requires = setuptools +setup_requires = setuptools +include_package_data = True +package_dir= + =src/engine +packages = find: + +[options.packages.find] + where=src/engine + +[options.entry_points] +console_scripts = + scons = SCons.Script.Main:main + + +[options.package_data] +* = *.txt, *.rst +SCons.Tool.docbook = *.* + + +[bdist_wheel] +universal=0 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..ff975d4 --- /dev/null +++ b/setup.py @@ -0,0 +1,6 @@ +from setuptools import setup,find_packages + +setup( + # packages = find_packages("src/engine"), + # package_dir={"":"src/engine"}, +) \ No newline at end of file -- cgit v0.12 From 23197f27e50f9f66d9f7263fbac5c38855731922 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 21 Mar 2020 13:43:26 -0700 Subject: First pass of refactor. Moved command line argument processing into site_scons/BuildCommandLine.py. Sorted out all changes caused by that. --- SConstruct | 242 +++++++++-------------------------------- doc/SConscript | 6 +- site_scons/BuildCommandLine.py | 144 ++++++++++++++++++++++++ site_scons/epydoc.py | 4 +- site_scons/site_init.py | 10 +- src/setup.py | 6 +- 6 files changed, 214 insertions(+), 198 deletions(-) create mode 100644 site_scons/BuildCommandLine.py diff --git a/SConstruct b/SConstruct index db71e0f..6a11bca 100644 --- a/SConstruct +++ b/SConstruct @@ -35,8 +35,6 @@ import fnmatch import os import os.path import sys -import time -import socket import textwrap import bootstrap @@ -45,88 +43,16 @@ project = 'scons' default_version = '3.1.2' copyright = "Copyright (c) %s The SCons Foundation" % copyright_years -SConsignFile() - # # We let the presence or absence of various utilities determine whether # or not we bother to build certain pieces of things. This should allow # people to still do SCons packaging work even if they don't have all # of the utilities installed # -gzip = whereis('gzip') -git = os.path.exists('.git') and whereis('git') -unzip = whereis('unzip') -zip = whereis('zip') - -# -# Now grab the information that we "build" into the files. -# -date = ARGUMENTS.get('DATE') -if not date: - date = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(int(os.environ.get('SOURCE_DATE_EPOCH', time.time())))) - -developer = ARGUMENTS.get('DEVELOPER') -if not developer: - for variable in ['USERNAME', 'LOGNAME', 'USER']: - developer = os.environ.get(variable) - if developer: - break - if os.environ.get('SOURCE_DATE_EPOCH'): - developer = '_reproducible' - -build_system = ARGUMENTS.get('BUILD_SYSTEM') -if not build_system: - if os.environ.get('SOURCE_DATE_EPOCH'): - build_system = '_reproducible' - else: - build_system = socket.gethostname().split('.')[0] - -version = ARGUMENTS.get('VERSION', '') -if not version: - version = default_version - -git_status_lines = [] - -if git: - cmd = "%s ls-files 2> /dev/null" % git - with os.popen(cmd, "r") as p: - git_status_lines = p.readlines() - -revision = ARGUMENTS.get('REVISION', '') - - -def generate_build_id(revision): - return revision - - -if not revision and git: - with os.popen("%s rev-parse HEAD 2> /dev/null" % git, "r") as p: - git_hash = p.read().strip() - - - def generate_build_id(revision): - result = git_hash - if [l for l in git_status_lines if 'modified' in l]: - result = result + '[MODIFIED]' - return result - - - revision = git_hash - -checkpoint = ARGUMENTS.get('CHECKPOINT', '') -if checkpoint: - if checkpoint == 'd': - checkpoint = time.strftime('%Y%m%d', time.localtime(time.time())) - elif checkpoint == 'r': - checkpoint = 'r' + revision - version = version + '.beta.' + checkpoint - -build_id = ARGUMENTS.get('BUILD_ID') -if build_id is None: - if revision: - build_id = generate_build_id(revision) - else: - build_id = '' +print("git :%s"%git) +print("gzip :%s"%gzip) +print("unzip :%s"%unzip) +print("zip :%s"%zip_path) # # Adding some paths to sys.path, this is mainly needed @@ -138,64 +64,14 @@ for a in addpaths: if a not in sys.path: sys.path.append(a) -# Re-exporting LD_LIBRARY_PATH is necessary if the Python version was -# built with the --enable-shared option. - -ENV = {'PATH': os.environ['PATH']} -for key in ['LOGNAME', 'PYTHONPATH', 'LD_LIBRARY_PATH']: - if key in os.environ: - ENV[key] = os.environ[key] - -build_dir = ARGUMENTS.get('BUILDDIR', 'build') -if not os.path.isabs(build_dir): - build_dir = os.path.normpath(os.path.join(os.getcwd(), build_dir)) - -command_line_variables = [ - ("BUILDDIR=", "The directory in which to build the packages. " + - "The default is the './build' subdirectory."), - - ("BUILD_ID=", "An identifier for the specific build." + - "The default is the Subversion revision number."), - - ("BUILD_SYSTEM=", "The system on which the packages were built. " + - "The default is whatever hostname is returned " + - "by socket.gethostname(). If SOURCE_DATE_EPOCH " + - "env var is set, '_reproducible' is the default."), - - ("CHECKPOINT=", "The specific checkpoint release being packaged, " + - "which will be appended to the VERSION string. " + - "A value of CHECKPOINT=d will generate a string " + - "of 'd' plus today's date in the format YYYMMDD. " + - "A value of CHECKPOINT=r will generate a " + - "string of 'r' plus the Subversion revision " + - "number. Any other CHECKPOINT= string will be " + - "used as is. There is no default value."), - - ("DATE=", "The date string representing when the packaging " + - "build occurred. The default is the day and time " + - "the SConstruct file was invoked, in the format " + - "YYYY/MM/DD HH:MM:SS."), - - ("DEVELOPER=", "The developer who created the packages. " + - "The default is the first set environment " + - "variable from the list $USERNAME, $LOGNAME, $USER." + - "If the SOURCE_DATE_EPOCH env var is set, " + - "'_reproducible' is the default."), - - ("REVISION=", "The revision number of the source being built. " + - "The default is the git hash returned " + - "'git rev-parse HEAD', with an appended string of " + - "'[MODIFIED]' if there are any changes in the " + - "working copy."), - - ("VERSION=", "The SCons version being packaged. The default " + - "is the hard-coded value '%s' " % default_version + - "from this SConstruct file."), - - ("SKIP_DOC=","Skip building all documents. The default is False (build docs)"), -] +command_line = BuildCommandLine(default_version) +command_line.process_command_line_vars() + + +Default('.', command_line.build_dir) +# Just make copies, don't symlink them. +SetOption('duplicate', 'copy') -Default('.', build_dir) packaging_flavors = [ ('tar-gz', "The normal .tar.gz file for end-user installation."), @@ -210,15 +86,15 @@ packaging_flavors = [ "(including tests and documentation)."), ] -test_tar_gz_dir = os.path.join(build_dir, "test-tar-gz") -test_src_tar_gz_dir = os.path.join(build_dir, "test-src-tar-gz") -test_local_tar_gz_dir = os.path.join(build_dir, "test-local-tar-gz") -test_zip_dir = os.path.join(build_dir, "test-zip") -test_src_zip_dir = os.path.join(build_dir, "test-src-zip") -test_local_zip_dir = os.path.join(build_dir, "test-local-zip") +test_tar_gz_dir = os.path.join(command_line.build_dir, "test-tar-gz") +test_src_tar_gz_dir = os.path.join(command_line.build_dir, "test-src-tar-gz") +test_local_tar_gz_dir = os.path.join(command_line.build_dir, "test-local-tar-gz") +test_zip_dir = os.path.join(command_line.build_dir, "test-zip") +test_src_zip_dir = os.path.join(command_line.build_dir, "test-src-zip") +test_local_zip_dir = os.path.join(command_line.build_dir, "test-local-zip") -unpack_tar_gz_dir = os.path.join(build_dir, "unpack-tar-gz") -unpack_zip_dir = os.path.join(build_dir, "unpack-zip") +unpack_tar_gz_dir = os.path.join(command_line.build_dir, "unpack-tar-gz") +unpack_zip_dir = os.path.join(command_line.build_dir, "unpack-zip") if is_windows(): tar_hflag = '' @@ -253,7 +129,7 @@ The following command-line variables can be set: """) -for variable, help_text in command_line_variables: +for variable, help_text in command_line.command_line_variables: tw = textwrap.TextWrapper( width=78, initial_indent=indent_fmt % variable, @@ -265,27 +141,26 @@ revaction = SCons_revision revbuilder = Builder(action=Action(SCons_revision, varlist=['COPYRIGHT', 'VERSION'])) -# Just make copies, don't symlink them. -SetOption('duplicate', 'copy') env = Environment( - ENV=ENV, + ENV=command_line.ENV, - BUILD=build_id, - BUILDDIR=build_dir, - BUILDSYS=build_system, + BUILD=command_line.build_id, + BUILDDIR=command_line.build_dir, + BUILDSYS=command_line.build_system, COPYRIGHT=copyright, - DATE=date, + DATE=command_line.date, DEB_DATE=deb_date, - DEVELOPER=developer, - DISTDIR=os.path.join(build_dir, 'dist'), + + DEVELOPER=command_line.developer, + DISTDIR=os.path.join(command_line.build_dir, 'dist'), MONTH_YEAR=month_year, - REVISION=revision, - VERSION=version, + REVISION=command_line.revision, + VERSION=command_line.version, TAR_HFLAG=tar_hflag, - ZIP=zip, + ZIP=zip_path, ZIPFLAGS='-r', UNZIP=unzip, UNZIPFLAGS='-o -d $UNPACK_ZIP_DIR', @@ -307,7 +182,7 @@ env = Environment( PYTHONFLAGS='-tt', ) -Version_values = [Value(version), Value(build_id)] +Version_values = [Value(command_line.version), Value(command_line.build_id)] # # Define SCons packages. @@ -423,13 +298,13 @@ for p in [scons]: # Initialize variables with the right directories for this package. # pkg = p['pkg'] - pkg_version = "%s-%s" % (pkg, version) + pkg_version = "%s-%s" % (pkg, command_line.version) src = 'src' if 'src_subdir' in p: src = os.path.join(src, p['src_subdir']) - build = os.path.join(build_dir, pkg) + build = os.path.join(command_line.build_dir, pkg) tar_gz = os.path.join(build, 'dist', "%s.tar.gz" % pkg_version) platform_tar_gz = os.path.join(build, @@ -616,8 +491,8 @@ for p in [scons]: # Generate portage files for submission to Gentoo Linux. # gentoo = os.path.join(build, 'gentoo') - ebuild = os.path.join(gentoo, 'scons-%s.ebuild' % version) - digest = os.path.join(gentoo, 'files', 'digest-scons-%s' % version) + ebuild = os.path.join(gentoo, 'scons-%s.ebuild' % command_line.version) + digest = os.path.join(gentoo, 'files', 'digest-scons-%s' % command_line.version) env.Command(ebuild, os.path.join('gentoo', 'scons.ebuild.in'), SCons_revision) @@ -711,11 +586,11 @@ for p in [scons]: # build their SCons-buildable packages without having to # install SCons. # - s_l_v = '%s-local-%s' % (pkg, version) + s_l_v = '%s-local-%s' % (pkg, command_line.version) local = pkg + '-local' - build_dir_local = os.path.join(build_dir, local) - build_dir_local_slv = os.path.join(build_dir, local, s_l_v) + build_dir_local = os.path.join(command_line.build_dir, local) + build_dir_local_slv = os.path.join(command_line.build_dir, local, s_l_v) dist_local_tar_gz = os.path.join("$DISTDIR/%s.tar.gz" % s_l_v) dist_local_zip = os.path.join("$DISTDIR/%s.zip" % s_l_v) @@ -785,23 +660,10 @@ for p in [scons]: # # # -Export('build_dir', 'env') - -SConscript('testing/framework/SConscript') - -# -# -# -sp = env.Install(build_dir, 'runtest.py') -Local(sp) -files = [ - 'runtest.py', -] - # # Documentation. # -Export('build_dir', 'env', 'whereis', 'revaction') +Export('command_line', 'env', 'whereis', 'revaction') SConscript('doc/SConscript') @@ -811,8 +673,8 @@ SConscript('doc/SConscript') # -sfiles = [l.split()[-1] for l in git_status_lines] -if git_status_lines: +sfiles = [l.split()[-1] for l in command_line.git_status_lines] +if command_line.git_status_lines: # slines = [l for l in git_status_lines if 'modified:' in l] # sfiles = [l.split()[-1] for l in slines] pass @@ -831,13 +693,13 @@ if sfiles: if sfiles: ps = "%s-src" % project - psv = "%s-%s" % (ps, version) - b_ps = os.path.join(build_dir, ps) - b_psv = os.path.join(build_dir, psv) + psv = "%s-%s" % (ps, command_line.version) + b_ps = os.path.join(command_line.build_dir, ps) + b_psv = os.path.join(command_line.build_dir, psv) b_psv_stamp = b_psv + '-stamp' - src_tar_gz = os.path.join(build_dir, 'dist', '%s.tar.gz' % psv) - src_zip = os.path.join(build_dir, 'dist', '%s.zip' % psv) + src_tar_gz = os.path.join(command_line.build_dir, 'dist', '%s.tar.gz' % psv) + src_zip = os.path.join(command_line.build_dir, 'dist', '%s.zip' % psv) Local(src_tar_gz, src_zip) @@ -900,7 +762,7 @@ if sfiles: scons_lib_dir = os.path.join(unpack_tar_gz_dir, psv, 'src', 'engine') ENV = env.Dictionary('ENV').copy() ENV['SCONS_LIB_DIR'] = scons_lib_dir - ENV['USERNAME'] = developer + ENV['USERNAME'] = command_line.developer env.Command(dfiles, unpack_tar_gz_files, [ Delete(os.path.join(unpack_tar_gz_dir, @@ -953,7 +815,7 @@ if sfiles: scons_lib_dir = os.path.join(unpack_zip_dir, psv, 'src', 'engine') ENV = env.Dictionary('ENV').copy() ENV['SCONS_LIB_DIR'] = scons_lib_dir - ENV['USERNAME'] = developer + ENV['USERNAME'] = command_line.developer env.Command(dfiles, unpack_zip_files, [ Delete(os.path.join(unpack_zip_dir, @@ -977,7 +839,7 @@ if sfiles: for pf, help_text in packaging_flavors: Alias(pf, [ - os.path.join(build_dir, 'test-' + pf), - os.path.join(build_dir, 'testing/framework'), - os.path.join(build_dir, 'runtest.py'), + os.path.join(command_line.build_dir, 'test-' + pf), + os.path.join(command_line.build_dir, 'testing/framework'), + os.path.join(command_line.build_dir, 'runtest.py'), ]) diff --git a/doc/SConscript b/doc/SConscript index 580f4ca..4e9fb74 100644 --- a/doc/SConscript +++ b/doc/SConscript @@ -31,7 +31,7 @@ import glob import bootstrap -Import('build_dir', 'env', 'whereis', 'revaction') +Import('command_line', 'env', 'whereis', 'revaction') # # -- Check prerequisites for building the documentation --- @@ -64,7 +64,7 @@ if skip_doc_arg is not None: # env = env.Clone() -build = os.path.join(build_dir, 'doc') +build = os.path.join(command_line.build_dir, 'doc') gs = whereis('gs') @@ -252,7 +252,7 @@ else: fpattern = [fpattern] if use_builddir: - target_dir = env.Dir(os.path.join(build_dir, *(toolpath+paths))) + target_dir = env.Dir(os.path.join(command_line.build_dir, *(toolpath+paths))) buildsuite.extend(env.GlobInstall(target_dir, os.path.join('..', *(toolpath+paths+fpattern)))) else: diff --git a/site_scons/BuildCommandLine.py b/site_scons/BuildCommandLine.py new file mode 100644 index 0000000..4694472 --- /dev/null +++ b/site_scons/BuildCommandLine.py @@ -0,0 +1,144 @@ +import time +import os +import socket + +from SCons.Script import ARGUMENTS + +class BuildCommandLine(object): + + git = None + + def init_command_line_variables(self): + self.command_line_variables = [ + ("BUILDDIR=", "The directory in which to build the packages. " + + "The default is the './build' subdirectory."), + + ("BUILD_ID=", "An identifier for the specific build." + + "The default is the Subversion revision number."), + + ("BUILD_SYSTEM=", "The system on which the packages were built. " + + "The default is whatever hostname is returned " + + "by socket.gethostname(). If SOURCE_DATE_EPOCH " + + "env var is set, '_reproducible' is the default."), + + ("CHECKPOINT=", "The specific checkpoint release being packaged, " + + "which will be appended to the VERSION string. " + + "A value of CHECKPOINT=d will generate a string " + + "of 'd' plus today's date in the format YYYMMDD. " + + "A value of CHECKPOINT=r will generate a " + + "string of 'r' plus the Subversion revision " + + "number. Any other CHECKPOINT= string will be " + + "used as is. There is no default value."), + + ("DATE=", "The date string representing when the packaging " + + "build occurred. The default is the day and time " + + "the SConstruct file was invoked, in the format " + + "YYYY/MM/DD HH:MM:SS."), + + ("DEVELOPER=", "The developer who created the packages. " + + "The default is the first set environment " + + "variable from the list $USERNAME, $LOGNAME, $USER." + + "If the SOURCE_DATE_EPOCH env var is set, " + + "'_reproducible' is the default."), + + ("REVISION=", "The revision number of the source being built. " + + "The default is the git hash returned " + + "'git rev-parse HEAD', with an appended string of " + + "'[MODIFIED]' if there are any changes in the " + + "working copy."), + + ("VERSION=", "The SCons version being packaged. The default " + + "is the hard-coded value '%s' " % self.default_version + + "from this SConstruct file."), + + ("SKIP_DOC=", "Skip building all documents. The default is False (build docs)"), + ] + + def __init__(self, default_version="99.99.99"): + self.date = None + self.default_version = default_version + self.developer = None + self.build_dir = None + self.build_system = None + self.version = None + self.revision = None + self.git_status_lines = [] + self.git_hash = None + + self.init_command_line_variables() + + def process_command_line_vars(self): + # + # Now grab the information that we "build" into the files. + # + self.date = ARGUMENTS.get('DATE') + if not self.date: + self.date = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(int(os.environ.get('SOURCE_DATE_EPOCH', time.time())))) + + self.developer = ARGUMENTS.get('DEVELOPER') + if not self.developer: + for variable in ['USERNAME', 'LOGNAME', 'USER']: + self.developer = os.environ.get(variable) + if self.developer: + break + if os.environ.get('SOURCE_DATE_EPOCH'): + self.developer = '_reproducible' + + self.build_system = ARGUMENTS.get('BUILD_SYSTEM') + if not self.build_system: + if os.environ.get('SOURCE_DATE_EPOCH'): + self.build_system = '_reproducible' + else: + self.build_system = socket.gethostname().split('.')[0] + + self.version = ARGUMENTS.get('VERSION', '') + if not self.version: + self.version = self.default_version + + if BuildCommandLine.git: + cmd = "%s ls-files 2> /dev/null" % BuildCommandLine.git + with os.popen(cmd, "r") as p: + self.git_status_lines = p.readlines() + + self.revision = ARGUMENTS.get('REVISION', '') + + def generate_build_id(revision): + return revision + + if not self.revision and BuildCommandLine.git: + with os.popen("%s rev-parse HEAD 2> /dev/null" % BuildCommandLine.git, "r") as p: + self.git_hash = p.read().strip() + + def generate_build_id(revision): + result = self.git_hash + if [l for l in self.git_status_lines if 'modified' in l]: + result = result + '[MODIFIED]' + return result + + self.revision = self.git_hash + + self.checkpoint = ARGUMENTS.get('CHECKPOINT', '') + if self.checkpoint: + if self.checkpoint == 'd': + cself.heckpoint = time.strftime('%Y%m%d', time.localtime(time.time())) + elif self.checkpoint == 'r': + self.checkpoint = 'r' + self.revision + self.version = self.version + '.beta.' + self.checkpoint + + self.build_id = ARGUMENTS.get('BUILD_ID') + if self.build_id is None: + if self.revision: + self.build_id = generate_build_id(self.revision) + else: + self.build_id = '' + + # Re-exporting LD_LIBRARY_PATH is necessary if the Python version was + # built with the --enable-shared option. + self.ENV = {'PATH': os.environ['PATH']} + for key in ['LOGNAME', 'PYTHONPATH', 'LD_LIBRARY_PATH']: + if key in os.environ: + self.ENV[key] = os.environ[key] + + self.build_dir = ARGUMENTS.get('BUILDDIR', 'build') + if not os.path.isabs(self.build_dir): + self.build_dir = os.path.normpath(os.path.join(os.getcwd(), self.build_dir)) diff --git a/site_scons/epydoc.py b/site_scons/epydoc.py index 149e9dc..da74d9c 100644 --- a/site_scons/epydoc.py +++ b/site_scons/epydoc.py @@ -28,7 +28,6 @@ from SCons.Script import Delete, Touch, WhereIs epydoc_cli = WhereIs('epydoc') - if not epydoc_cli: try: import epydoc @@ -92,7 +91,8 @@ if not epydoc_cli: Touch('$TARGET'), ] -else: # epydoc_cli is found +else: + # epydoc_cli is found epydoc_commands = [ Delete('$OUTDIR'), '$EPYDOC $EPYDOCFLAGS --debug --output $OUTDIR --docformat=restructuredText --name SCons --url http://www.scons.org/ $SOURCES', diff --git a/site_scons/site_init.py b/site_scons/site_init.py index d4df473..7e7c569 100644 --- a/site_scons/site_init.py +++ b/site_scons/site_init.py @@ -2,4 +2,12 @@ from SConsRevision import SCons_revision from Utilities import is_windows, whereis, platform, deb_date from zip_utils import unzipit, zipit, zcat from soe_utils import soelim, soscan, soelimbuilder -from epydoc import epydoc_cli, epydoc_commands \ No newline at end of file +from epydoc import epydoc_cli, epydoc_commands +from BuildCommandLine import BuildCommandLine + +gzip = whereis('gzip') +git = os.path.exists('.git') and whereis('git') +unzip = whereis('unzip') +zip_path = whereis('zip') + +BuildCommandLine.git = git \ No newline at end of file diff --git a/src/setup.py b/src/setup.py index 261e2a4..bea888b 100755 --- a/src/setup.py +++ b/src/setup.py @@ -27,7 +27,7 @@ import distutils.command.install_data import distutils.command.install import distutils.core import distutils -# import setuptools +import setuptools """ NOTE: Installed SCons is not importable like usual Python packages. It is executed explicitly with command line scripts. This allows multiple @@ -165,7 +165,6 @@ class install(_install): self.install_bat = 0 self.no_install_bat = False # not is_win32 self.install_man = 0 - self.no_install_man = is_win32 self.standard_lib = 0 self.standalone_lib = 0 self.version_lib = 0 @@ -505,6 +504,9 @@ arguments = { 'install_data': install_data, 'install_scripts': install_scripts, 'build_scripts': build_scripts}, + 'install_requires' : { + "pywin32;platform_system=='Windows'" + } } distutils.core.setup(**arguments) -- cgit v0.12 From 072ebad9f866bdb4b79f5309071c0a92df1e21be Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 21 Mar 2020 16:35:56 -0700 Subject: Move scripts from src/scripts to scripts. Simplify scons.py to only be used for running from this source tree and for scons-local packaging --- scripts/MANIFEST.in | 4 + scripts/scons-configure-cache.py | 177 +++++ scripts/scons-time.py | 1480 +++++++++++++++++++++++++++++++++++ scripts/scons.bat | 38 + scripts/scons.py | 102 +++ scripts/sconsign.py | 652 +++++++++++++++ src/script/MANIFEST.in | 4 - src/script/scons-configure-cache.py | 177 ----- src/script/scons-time.py | 1480 ----------------------------------- src/script/scons.bat | 38 - src/script/scons.py | 208 ----- src/script/sconsign.py | 652 --------------- 12 files changed, 2453 insertions(+), 2559 deletions(-) create mode 100644 scripts/MANIFEST.in create mode 100644 scripts/scons-configure-cache.py create mode 100644 scripts/scons-time.py create mode 100644 scripts/scons.bat create mode 100755 scripts/scons.py create mode 100644 scripts/sconsign.py delete mode 100644 src/script/MANIFEST.in delete mode 100644 src/script/scons-configure-cache.py delete mode 100644 src/script/scons-time.py delete mode 100644 src/script/scons.bat delete mode 100755 src/script/scons.py delete mode 100644 src/script/sconsign.py diff --git a/scripts/MANIFEST.in b/scripts/MANIFEST.in new file mode 100644 index 0000000..d10cc82 --- /dev/null +++ b/scripts/MANIFEST.in @@ -0,0 +1,4 @@ +scons +sconsign +scons-time +scons-configure-cache diff --git a/scripts/scons-configure-cache.py b/scripts/scons-configure-cache.py new file mode 100644 index 0000000..716315c --- /dev/null +++ b/scripts/scons-configure-cache.py @@ -0,0 +1,177 @@ +#! /usr/bin/env python +# +# SCons - a Software Constructor +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +"""Show or convert the configuration of an SCons cache directory. + +A cache of derived files is stored by file signature. +The files are split into directories named by the first few +digits of the signature. The prefix length used for directory +names can be changed by this script. +""" + +import argparse +import glob +import json +import os + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +__version__ = "__VERSION__" + +__build__ = "__BUILD__" + +__buildsys__ = "__BUILDSYS__" + +__date__ = "__DATE__" + +__developer__ = "__DEVELOPER__" + + +def rearrange_cache_entries(current_prefix_len, new_prefix_len): + """Move cache files if prefix length changed. + + Move the existing cache files to new directories of the + appropriate name length and clean up the old directories. + """ + print('Changing prefix length from', current_prefix_len, + 'to', new_prefix_len) + dirs = set() + old_dirs = set() + for file in glob.iglob(os.path.join('*', '*')): + name = os.path.basename(file) + dname = name[:current_prefix_len].upper() + if dname not in old_dirs: + print('Migrating', dname) + old_dirs.add(dname) + dname = name[:new_prefix_len].upper() + if dname not in dirs: + os.mkdir(dname) + dirs.add(dname) + os.rename(file, os.path.join(dname, name)) + + # Now delete the original directories + for dname in old_dirs: + os.rmdir(dname) + + +# The configuration dictionary should have one entry per entry in the +# cache config. The value of each entry should include the following: +# implicit - (optional) This is to allow adding a new config entry and also +# changing the behaviour of the system at the same time. This +# indicates the value the config entry would have had if it had +# been specified. +# default - The value the config entry should have if it wasn't previously +# specified +# command-line - parameters to pass to ArgumentParser.add_argument +# converter - (optional) Function to call if conversion is required +# if this configuration entry changes +config_entries = { + 'prefix_len': { + 'implicit': 1, + 'default': 2, + 'command-line': { + 'help': 'Length of cache file name used as subdirectory prefix', + 'metavar': '', + 'type': int + }, + 'converter': rearrange_cache_entries + } +} + +parser = argparse.ArgumentParser( + description='Modify the configuration of an scons cache directory', + epilog=''' + Unspecified options will not be changed unless they are not + set at all, in which case they are set to an appropriate default. + ''') + +parser.add_argument('cache-dir', help='Path to scons cache directory') +for param in config_entries: + parser.add_argument('--' + param.replace('_', '-'), + **config_entries[param]['command-line']) +parser.add_argument('--version', + action='version', + version='%(prog)s 1.0') +parser.add_argument('--show', + action="store_true", + help="show current configuration") + +# Get the command line as a dict without any of the unspecified entries. +args = dict([x for x in vars(parser.parse_args()).items() if x[1]]) + +# It seems somewhat strange to me, but positional arguments don't get the - +# in the name changed to _, whereas optional arguments do... +cache = args['cache-dir'] +if not os.path.isdir(cache): + raise RuntimeError("There is no cache directory named %s" % cache) +os.chdir(cache) +del args['cache-dir'] + +if not os.path.exists('config'): + # old config dirs did not have a 'config' file. Try to update. + # Validate the only files in the directory are directories 0-9, a-f + expected = ['{:X}'.format(x) for x in range(0, 16)] + if not set(os.listdir('.')).issubset(expected): + raise RuntimeError( + "%s does not look like a valid version 1 cache directory" % cache) + config = dict() +else: + with open('config') as conf: + config = json.load(conf) + +if args.get('show', None): + print("Current configuration in '%s':" % cache) + print(json.dumps(config, sort_keys=True, + indent=4, separators=(',', ': '))) + # in case of the show argument, emit some stats as well + file_count = 0 + for _, _, files in os.walk('.'): + file_count += len(files) + if file_count: # skip config file if it exists + file_count -= 1 + print("Cache contains %s files" % file_count) + del args['show'] + +# Find any keys that are not currently set but should be +for key in config_entries: + if key not in config: + if 'implicit' in config_entries[key]: + config[key] = config_entries[key]['implicit'] + else: + config[key] = config_entries[key]['default'] + if key not in args: + args[key] = config_entries[key]['default'] + +# Now go through each entry in args to see if it changes an existing config +# setting. +for key in args: + if args[key] != config[key]: + if 'converter' in config_entries[key]: + config_entries[key]['converter'](config[key], args[key]) + config[key] = args[key] + +# and write the updated config file +with open('config', 'w') as conf: + json.dump(config, conf) diff --git a/scripts/scons-time.py b/scripts/scons-time.py new file mode 100644 index 0000000..e4dd863 --- /dev/null +++ b/scripts/scons-time.py @@ -0,0 +1,1480 @@ +#!/usr/bin/env python +# +# scons-time - run SCons timings and collect statistics +# +# A script for running a configuration through SCons with a standard +# set of invocations to collect timing and memory statistics and to +# capture the results in a consistent set of output files for display +# and analysis. +# + +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +import getopt +import glob +import os +import re +import shutil +import sys +import tempfile +import time +import subprocess + +def HACK_for_exec(cmd, *args): + """ + For some reason, Python won't allow an exec() within a function + that also declares an internal function (including lambda functions). + This function is a hack that calls exec() in a function with no + internal functions. + """ + if not args: exec(cmd) + elif len(args) == 1: exec(cmd, args[0]) + else: exec(cmd, args[0], args[1]) + +class Plotter(object): + def increment_size(self, largest): + """ + Return the size of each horizontal increment line for a specified + maximum value. This returns a value that will provide somewhere + between 5 and 9 horizontal lines on the graph, on some set of + boundaries that are multiples of 10/100/1000/etc. + """ + i = largest // 5 + if not i: + return largest + multiplier = 1 + while i >= 10: + i = i // 10 + multiplier = multiplier * 10 + return i * multiplier + + def max_graph_value(self, largest): + # Round up to next integer. + largest = int(largest) + 1 + increment = self.increment_size(largest) + return ((largest + increment - 1) // increment) * increment + +class Line(object): + def __init__(self, points, type, title, label, comment, fmt="%s %s"): + self.points = points + self.type = type + self.title = title + self.label = label + self.comment = comment + self.fmt = fmt + + def print_label(self, inx, x, y): + if self.label: + print('set label %s "%s" at %0.1f,%0.1f right' % (inx, self.label, x, y)) + + def plot_string(self): + if self.title: + title_string = 'title "%s"' % self.title + else: + title_string = 'notitle' + return "'-' %s with lines lt %s" % (title_string, self.type) + + def print_points(self, fmt=None): + if fmt is None: + fmt = self.fmt + if self.comment: + print('# %s' % self.comment) + for x, y in self.points: + # If y is None, it usually represents some kind of break + # in the line's index number. We might want to represent + # this some way rather than just drawing the line straight + # between the two points on either side. + if y is not None: + print(fmt % (x, y)) + print('e') + + def get_x_values(self): + return [ p[0] for p in self.points ] + + def get_y_values(self): + return [ p[1] for p in self.points ] + +class Gnuplotter(Plotter): + + def __init__(self, title, key_location): + self.lines = [] + self.title = title + self.key_location = key_location + + def line(self, points, type, title=None, label=None, comment=None, fmt='%s %s'): + if points: + line = Line(points, type, title, label, comment, fmt) + self.lines.append(line) + + def plot_string(self, line): + return line.plot_string() + + def vertical_bar(self, x, type, label, comment): + if self.get_min_x() <= x <= self.get_max_x(): + points = [(x, 0), (x, self.max_graph_value(self.get_max_y()))] + self.line(points, type, label, comment) + + def get_all_x_values(self): + result = [] + for line in self.lines: + result.extend(line.get_x_values()) + return [r for r in result if r is not None] + + def get_all_y_values(self): + result = [] + for line in self.lines: + result.extend(line.get_y_values()) + return [r for r in result if r is not None] + + def get_min_x(self): + try: + return self.min_x + except AttributeError: + try: + self.min_x = min(self.get_all_x_values()) + except ValueError: + self.min_x = 0 + return self.min_x + + def get_max_x(self): + try: + return self.max_x + except AttributeError: + try: + self.max_x = max(self.get_all_x_values()) + except ValueError: + self.max_x = 0 + return self.max_x + + def get_min_y(self): + try: + return self.min_y + except AttributeError: + try: + self.min_y = min(self.get_all_y_values()) + except ValueError: + self.min_y = 0 + return self.min_y + + def get_max_y(self): + try: + return self.max_y + except AttributeError: + try: + self.max_y = max(self.get_all_y_values()) + except ValueError: + self.max_y = 0 + return self.max_y + + def draw(self): + + if not self.lines: + return + + if self.title: + print('set title "%s"' % self.title) + print('set key %s' % self.key_location) + + min_y = self.get_min_y() + max_y = self.max_graph_value(self.get_max_y()) + incr = (max_y - min_y) / 10.0 + start = min_y + (max_y / 2.0) + (2.0 * incr) + position = [ start - (i * incr) for i in range(5) ] + + inx = 1 + for line in self.lines: + line.print_label(inx, line.points[0][0]-1, + position[(inx-1) % len(position)]) + inx += 1 + + plot_strings = [ self.plot_string(l) for l in self.lines ] + print('plot ' + ', \\\n '.join(plot_strings)) + + for line in self.lines: + line.print_points() + + + +def untar(fname): + import tarfile + tar = tarfile.open(name=fname, mode='r') + for tarinfo in tar: + tar.extract(tarinfo) + tar.close() + +def unzip(fname): + import zipfile + zf = zipfile.ZipFile(fname, 'r') + for name in zf.namelist(): + dir = os.path.dirname(name) + try: + os.makedirs(dir) + except: + pass + with open(name, 'wb') as f: + f.write(zf.read(name)) + +def read_tree(dir): + for dirpath, dirnames, filenames in os.walk(dir): + for fn in filenames: + fn = os.path.join(dirpath, fn) + if os.path.isfile(fn): + with open(fn, 'rb') as f: + f.read() + +def redirect_to_file(command, log): + return '%s > %s 2>&1' % (command, log) + +def tee_to_file(command, log): + return '%s 2>&1 | tee %s' % (command, log) + + + +class SConsTimer(object): + """ + Usage: scons-time SUBCOMMAND [ARGUMENTS] + Type "scons-time help SUBCOMMAND" for help on a specific subcommand. + + Available subcommands: + func Extract test-run data for a function + help Provides help + mem Extract --debug=memory data from test runs + obj Extract --debug=count data from test runs + time Extract --debug=time data from test runs + run Runs a test configuration + """ + + name = 'scons-time' + name_spaces = ' '*len(name) + + def makedict(**kw): + return kw + + default_settings = makedict( + chdir = None, + config_file = None, + initial_commands = [], + key_location = 'bottom left', + orig_cwd = os.getcwd(), + outdir = None, + prefix = '', + python = '"%s"' % sys.executable, + redirect = redirect_to_file, + scons = None, + scons_flags = '--debug=count --debug=memory --debug=time --debug=memoizer', + scons_lib_dir = None, + scons_wrapper = None, + startup_targets = '--help', + subdir = None, + subversion_url = None, + svn = 'svn', + svn_co_flag = '-q', + tar = 'tar', + targets = '', + targets0 = None, + targets1 = None, + targets2 = None, + title = None, + unzip = 'unzip', + verbose = False, + vertical_bars = [], + + unpack_map = { + '.tar.gz' : (untar, '%(tar)s xzf %%s'), + '.tgz' : (untar, '%(tar)s xzf %%s'), + '.tar' : (untar, '%(tar)s xf %%s'), + '.zip' : (unzip, '%(unzip)s %%s'), + }, + ) + + run_titles = [ + 'Startup', + 'Full build', + 'Up-to-date build', + ] + + run_commands = [ + '%(python)s %(scons_wrapper)s %(scons_flags)s --profile=%(prof0)s %(targets0)s', + '%(python)s %(scons_wrapper)s %(scons_flags)s --profile=%(prof1)s %(targets1)s', + '%(python)s %(scons_wrapper)s %(scons_flags)s --profile=%(prof2)s %(targets2)s', + ] + + stages = [ + 'pre-read', + 'post-read', + 'pre-build', + 'post-build', + ] + + stage_strings = { + 'pre-read' : 'Memory before reading SConscript files:', + 'post-read' : 'Memory after reading SConscript files:', + 'pre-build' : 'Memory before building targets:', + 'post-build' : 'Memory after building targets:', + } + + memory_string_all = 'Memory ' + + default_stage = stages[-1] + + time_strings = { + 'total' : 'Total build time', + 'SConscripts' : 'Total SConscript file execution time', + 'SCons' : 'Total SCons execution time', + 'commands' : 'Total command execution time', + } + + time_string_all = 'Total .* time' + + # + + def __init__(self): + self.__dict__.update(self.default_settings) + + # Functions for displaying and executing commands. + + def subst(self, x, dictionary): + try: + return x % dictionary + except TypeError: + # x isn't a string (it's probably a Python function), + # so just return it. + return x + + def subst_variables(self, command, dictionary): + """ + Substitutes (via the format operator) the values in the specified + dictionary into the specified command. + + The command can be an (action, string) tuple. In all cases, we + perform substitution on strings and don't worry if something isn't + a string. (It's probably a Python function to be executed.) + """ + try: + command + '' + except TypeError: + action = command[0] + string = command[1] + args = command[2:] + else: + action = command + string = action + args = (()) + action = self.subst(action, dictionary) + string = self.subst(string, dictionary) + return (action, string, args) + + def _do_not_display(self, msg, *args): + pass + + def display(self, msg, *args): + """ + Displays the specified message. + + Each message is prepended with a standard prefix of our name + plus the time. + """ + if callable(msg): + msg = msg(*args) + else: + msg = msg % args + if msg is None: + return + fmt = '%s[%s]: %s\n' + sys.stdout.write(fmt % (self.name, time.strftime('%H:%M:%S'), msg)) + + def _do_not_execute(self, action, *args): + pass + + def execute(self, action, *args): + """ + Executes the specified action. + + The action is called if it's a callable Python function, and + otherwise passed to os.system(). + """ + if callable(action): + action(*args) + else: + os.system(action % args) + + def run_command_list(self, commands, dict): + """ + Executes a list of commands, substituting values from the + specified dictionary. + """ + commands = [ self.subst_variables(c, dict) for c in commands ] + for action, string, args in commands: + self.display(string, *args) + sys.stdout.flush() + status = self.execute(action, *args) + if status: + sys.exit(status) + + def log_display(self, command, log): + command = self.subst(command, self.__dict__) + if log: + command = self.redirect(command, log) + return command + + def log_execute(self, command, log): + command = self.subst(command, self.__dict__) + p = os.popen(command) + output = p.read() + p.close() + #TODO: convert to subrocess, os.popen is obsolete. This didn't work: + #process = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True) + #output = process.stdout.read() + #process.stdout.close() + #process.wait() + if self.verbose: + sys.stdout.write(output) + # TODO: Figure out + # Not sure we need to write binary here + with open(log, 'w') as f: + f.write(str(output)) + + def archive_splitext(self, path): + """ + Splits an archive name into a filename base and extension. + + This is like os.path.splitext() (which it calls) except that it + also looks for '.tar.gz' and treats it as an atomic extensions. + """ + if path.endswith('.tar.gz'): + return path[:-7], path[-7:] + else: + return os.path.splitext(path) + + def args_to_files(self, args, tail=None): + """ + Takes a list of arguments, expands any glob patterns, and + returns the last "tail" files from the list. + """ + files = [] + for a in args: + files.extend(sorted(glob.glob(a))) + + if tail: + files = files[-tail:] + + return files + + def ascii_table(self, files, columns, + line_function, file_function=lambda x: x, + *args, **kw): + + header_fmt = ' '.join(['%12s'] * len(columns)) + line_fmt = header_fmt + ' %s' + + print(header_fmt % columns) + + for file in files: + t = line_function(file, *args, **kw) + if t is None: + t = [] + diff = len(columns) - len(t) + if diff > 0: + t += [''] * diff + t.append(file_function(file)) + print(line_fmt % tuple(t)) + + def collect_results(self, files, function, *args, **kw): + results = {} + + for file in files: + base = os.path.splitext(file)[0] + run, index = base.split('-')[-2:] + + run = int(run) + index = int(index) + + value = function(file, *args, **kw) + + try: + r = results[index] + except KeyError: + r = [] + results[index] = r + r.append((run, value)) + + return results + + def doc_to_help(self, obj): + """ + Translates an object's __doc__ string into help text. + + This strips a consistent number of spaces from each line in the + help text, essentially "outdenting" the text to the left-most + column. + """ + doc = obj.__doc__ + if doc is None: + return '' + return self.outdent(doc) + + def find_next_run_number(self, dir, prefix): + """ + Returns the next run number in a directory for the specified prefix. + + Examines the contents the specified directory for files with the + specified prefix, extracts the run numbers from each file name, + and returns the next run number after the largest it finds. + """ + x = re.compile(re.escape(prefix) + '-([0-9]+).*') + matches = [x.match(e) for e in os.listdir(dir)] + matches = [_f for _f in matches if _f] + if not matches: + return 0 + run_numbers = [int(m.group(1)) for m in matches] + return int(max(run_numbers)) + 1 + + def gnuplot_results(self, results, fmt='%s %.3f'): + """ + Prints out a set of results in Gnuplot format. + """ + gp = Gnuplotter(self.title, self.key_location) + + for i in sorted(results.keys()): + try: + t = self.run_titles[i] + except IndexError: + t = '??? %s ???' % i + results[i].sort() + gp.line(results[i], i+1, t, None, t, fmt=fmt) + + for bar_tuple in self.vertical_bars: + try: + x, type, label, comment = bar_tuple + except ValueError: + x, type, label = bar_tuple + comment = label + gp.vertical_bar(x, type, label, comment) + + gp.draw() + + def logfile_name(self, invocation): + """ + Returns the absolute path of a log file for the specificed + invocation number. + """ + name = self.prefix_run + '-%d.log' % invocation + return os.path.join(self.outdir, name) + + def outdent(self, s): + """ + Strip as many spaces from each line as are found at the beginning + of the first line in the list. + """ + lines = s.split('\n') + if lines[0] == '': + lines = lines[1:] + spaces = re.match(' *', lines[0]).group(0) + def strip_initial_spaces(l, s=spaces): + if l.startswith(spaces): + l = l[len(spaces):] + return l + return '\n'.join([ strip_initial_spaces(l) for l in lines ]) + '\n' + + def profile_name(self, invocation): + """ + Returns the absolute path of a profile file for the specified + invocation number. + """ + name = self.prefix_run + '-%d.prof' % invocation + return os.path.join(self.outdir, name) + + def set_env(self, key, value): + os.environ[key] = value + + # + + def get_debug_times(self, file, time_string=None): + """ + Fetch times from the --debug=time strings in the specified file. + """ + if time_string is None: + search_string = self.time_string_all + else: + search_string = time_string + with open(file) as f: + contents = f.read() + if not contents: + sys.stderr.write('file %s has no contents!\n' % repr(file)) + return None + result = re.findall(r'%s: ([\d.]*)' % search_string, contents)[-4:] + result = [ float(r) for r in result ] + if time_string is not None: + try: + result = result[0] + except IndexError: + sys.stderr.write('file %s has no results!\n' % repr(file)) + return None + return result + + def get_function_profile(self, file, function): + """ + Returns the file, line number, function name, and cumulative time. + """ + try: + import pstats + except ImportError as e: + sys.stderr.write('%s: func: %s\n' % (self.name, e)) + sys.stderr.write('%s This version of Python is missing the profiler.\n' % self.name_spaces) + sys.stderr.write('%s Cannot use the "func" subcommand.\n' % self.name_spaces) + sys.exit(1) + statistics = pstats.Stats(file).stats + matches = [ e for e in statistics.items() if e[0][2] == function ] + r = matches[0] + return r[0][0], r[0][1], r[0][2], r[1][3] + + def get_function_time(self, file, function): + """ + Returns just the cumulative time for the specified function. + """ + return self.get_function_profile(file, function)[3] + + def get_memory(self, file, memory_string=None): + """ + Returns a list of integers of the amount of memory used. The + default behavior is to return all the stages. + """ + if memory_string is None: + search_string = self.memory_string_all + else: + search_string = memory_string + with open(file) as f: + lines = f.readlines() + lines = [ l for l in lines if l.startswith(search_string) ][-4:] + result = [ int(l.split()[-1]) for l in lines[-4:] ] + if len(result) == 1: + result = result[0] + return result + + def get_object_counts(self, file, object_name, index=None): + """ + Returns the counts of the specified object_name. + """ + object_string = ' ' + object_name + '\n' + with open(file) as f: + lines = f.readlines() + line = [ l for l in lines if l.endswith(object_string) ][0] + result = [ int(field) for field in line.split()[:4] ] + if index is not None: + result = result[index] + return result + + + command_alias = {} + + def execute_subcommand(self, argv): + """ + Executes the do_*() function for the specified subcommand (argv[0]). + """ + if not argv: + return + cmdName = self.command_alias.get(argv[0], argv[0]) + try: + func = getattr(self, 'do_' + cmdName) + except AttributeError: + return self.default(argv) + try: + return func(argv) + except TypeError as e: + sys.stderr.write("%s %s: %s\n" % (self.name, cmdName, e)) + import traceback + traceback.print_exc(file=sys.stderr) + sys.stderr.write("Try '%s help %s'\n" % (self.name, cmdName)) + + def default(self, argv): + """ + The default behavior for an unknown subcommand. Prints an + error message and exits. + """ + sys.stderr.write('%s: Unknown subcommand "%s".\n' % (self.name, argv[0])) + sys.stderr.write('Type "%s help" for usage.\n' % self.name) + sys.exit(1) + + # + + def do_help(self, argv): + """ + """ + if argv[1:]: + for arg in argv[1:]: + try: + func = getattr(self, 'do_' + arg) + except AttributeError: + sys.stderr.write('%s: No help for "%s"\n' % (self.name, arg)) + else: + try: + help = getattr(self, 'help_' + arg) + except AttributeError: + sys.stdout.write(self.doc_to_help(func)) + sys.stdout.flush() + else: + help() + else: + doc = self.doc_to_help(self.__class__) + if doc: + sys.stdout.write(doc) + sys.stdout.flush() + return None + + # + + def help_func(self): + help = """\ + Usage: scons-time func [OPTIONS] FILE [...] + + -C DIR, --chdir=DIR Change to DIR before looking for files + -f FILE, --file=FILE Read configuration from specified FILE + --fmt=FORMAT, --format=FORMAT Print data in specified FORMAT + --func=NAME, --function=NAME Report time for function NAME + -h, --help Print this help and exit + -p STRING, --prefix=STRING Use STRING as log file/profile prefix + -t NUMBER, --tail=NUMBER Only report the last NUMBER files + --title=TITLE Specify the output plot TITLE + """ + sys.stdout.write(self.outdent(help)) + sys.stdout.flush() + + def do_func(self, argv): + """ + """ + format = 'ascii' + function_name = '_main' + tail = None + + short_opts = '?C:f:hp:t:' + + long_opts = [ + 'chdir=', + 'file=', + 'fmt=', + 'format=', + 'func=', + 'function=', + 'help', + 'prefix=', + 'tail=', + 'title=', + ] + + opts, args = getopt.getopt(argv[1:], short_opts, long_opts) + + for o, a in opts: + if o in ('-C', '--chdir'): + self.chdir = a + elif o in ('-f', '--file'): + self.config_file = a + elif o in ('--fmt', '--format'): + format = a + elif o in ('--func', '--function'): + function_name = a + elif o in ('-?', '-h', '--help'): + self.do_help(['help', 'func']) + sys.exit(0) + elif o in ('--max',): + max_time = int(a) + elif o in ('-p', '--prefix'): + self.prefix = a + elif o in ('-t', '--tail'): + tail = int(a) + elif o in ('--title',): + self.title = a + + if self.config_file: + with open(self.config_file, 'r') as f: + config = f.read() + exec(config, self.__dict__) + + if self.chdir: + os.chdir(self.chdir) + + if not args: + + pattern = '%s*.prof' % self.prefix + args = self.args_to_files([pattern], tail) + + if not args: + if self.chdir: + directory = self.chdir + else: + directory = os.getcwd() + + sys.stderr.write('%s: func: No arguments specified.\n' % self.name) + sys.stderr.write('%s No %s*.prof files found in "%s".\n' % (self.name_spaces, self.prefix, directory)) + sys.stderr.write('%s Type "%s help func" for help.\n' % (self.name_spaces, self.name)) + sys.exit(1) + + else: + + args = self.args_to_files(args, tail) + + cwd_ = os.getcwd() + os.sep + + if format == 'ascii': + + for file in args: + try: + f, line, func, time = \ + self.get_function_profile(file, function_name) + except ValueError as e: + sys.stderr.write("%s: func: %s: %s\n" % + (self.name, file, e)) + else: + if f.startswith(cwd_): + f = f[len(cwd_):] + print("%.3f %s:%d(%s)" % (time, f, line, func)) + + elif format == 'gnuplot': + + results = self.collect_results(args, self.get_function_time, + function_name) + + self.gnuplot_results(results) + + else: + + sys.stderr.write('%s: func: Unknown format "%s".\n' % (self.name, format)) + sys.exit(1) + + # + + def help_mem(self): + help = """\ + Usage: scons-time mem [OPTIONS] FILE [...] + + -C DIR, --chdir=DIR Change to DIR before looking for files + -f FILE, --file=FILE Read configuration from specified FILE + --fmt=FORMAT, --format=FORMAT Print data in specified FORMAT + -h, --help Print this help and exit + -p STRING, --prefix=STRING Use STRING as log file/profile prefix + --stage=STAGE Plot memory at the specified stage: + pre-read, post-read, pre-build, + post-build (default: post-build) + -t NUMBER, --tail=NUMBER Only report the last NUMBER files + --title=TITLE Specify the output plot TITLE + """ + sys.stdout.write(self.outdent(help)) + sys.stdout.flush() + + def do_mem(self, argv): + + format = 'ascii' + logfile_path = lambda x: x + stage = self.default_stage + tail = None + + short_opts = '?C:f:hp:t:' + + long_opts = [ + 'chdir=', + 'file=', + 'fmt=', + 'format=', + 'help', + 'prefix=', + 'stage=', + 'tail=', + 'title=', + ] + + opts, args = getopt.getopt(argv[1:], short_opts, long_opts) + + for o, a in opts: + if o in ('-C', '--chdir'): + self.chdir = a + elif o in ('-f', '--file'): + self.config_file = a + elif o in ('--fmt', '--format'): + format = a + elif o in ('-?', '-h', '--help'): + self.do_help(['help', 'mem']) + sys.exit(0) + elif o in ('-p', '--prefix'): + self.prefix = a + elif o in ('--stage',): + if a not in self.stages: + sys.stderr.write('%s: mem: Unrecognized stage "%s".\n' % (self.name, a)) + sys.exit(1) + stage = a + elif o in ('-t', '--tail'): + tail = int(a) + elif o in ('--title',): + self.title = a + + if self.config_file: + with open(self.config_file, 'r') as f: + config = f.read() + HACK_for_exec(config, self.__dict__) + + if self.chdir: + os.chdir(self.chdir) + logfile_path = lambda x: os.path.join(self.chdir, x) + + if not args: + + pattern = '%s*.log' % self.prefix + args = self.args_to_files([pattern], tail) + + if not args: + if self.chdir: + directory = self.chdir + else: + directory = os.getcwd() + + sys.stderr.write('%s: mem: No arguments specified.\n' % self.name) + sys.stderr.write('%s No %s*.log files found in "%s".\n' % (self.name_spaces, self.prefix, directory)) + sys.stderr.write('%s Type "%s help mem" for help.\n' % (self.name_spaces, self.name)) + sys.exit(1) + + else: + + args = self.args_to_files(args, tail) + + cwd_ = os.getcwd() + os.sep + + if format == 'ascii': + + self.ascii_table(args, tuple(self.stages), self.get_memory, logfile_path) + + elif format == 'gnuplot': + + results = self.collect_results(args, self.get_memory, + self.stage_strings[stage]) + + self.gnuplot_results(results) + + else: + + sys.stderr.write('%s: mem: Unknown format "%s".\n' % (self.name, format)) + sys.exit(1) + + return 0 + + # + + def help_obj(self): + help = """\ + Usage: scons-time obj [OPTIONS] OBJECT FILE [...] + + -C DIR, --chdir=DIR Change to DIR before looking for files + -f FILE, --file=FILE Read configuration from specified FILE + --fmt=FORMAT, --format=FORMAT Print data in specified FORMAT + -h, --help Print this help and exit + -p STRING, --prefix=STRING Use STRING as log file/profile prefix + --stage=STAGE Plot memory at the specified stage: + pre-read, post-read, pre-build, + post-build (default: post-build) + -t NUMBER, --tail=NUMBER Only report the last NUMBER files + --title=TITLE Specify the output plot TITLE + """ + sys.stdout.write(self.outdent(help)) + sys.stdout.flush() + + def do_obj(self, argv): + + format = 'ascii' + logfile_path = lambda x: x + stage = self.default_stage + tail = None + + short_opts = '?C:f:hp:t:' + + long_opts = [ + 'chdir=', + 'file=', + 'fmt=', + 'format=', + 'help', + 'prefix=', + 'stage=', + 'tail=', + 'title=', + ] + + opts, args = getopt.getopt(argv[1:], short_opts, long_opts) + + for o, a in opts: + if o in ('-C', '--chdir'): + self.chdir = a + elif o in ('-f', '--file'): + self.config_file = a + elif o in ('--fmt', '--format'): + format = a + elif o in ('-?', '-h', '--help'): + self.do_help(['help', 'obj']) + sys.exit(0) + elif o in ('-p', '--prefix'): + self.prefix = a + elif o in ('--stage',): + if a not in self.stages: + sys.stderr.write('%s: obj: Unrecognized stage "%s".\n' % (self.name, a)) + sys.stderr.write('%s Type "%s help obj" for help.\n' % (self.name_spaces, self.name)) + sys.exit(1) + stage = a + elif o in ('-t', '--tail'): + tail = int(a) + elif o in ('--title',): + self.title = a + + if not args: + sys.stderr.write('%s: obj: Must specify an object name.\n' % self.name) + sys.stderr.write('%s Type "%s help obj" for help.\n' % (self.name_spaces, self.name)) + sys.exit(1) + + object_name = args.pop(0) + + if self.config_file: + with open(self.config_file, 'r') as f: + config = f.read() + HACK_for_exec(config, self.__dict__) + + if self.chdir: + os.chdir(self.chdir) + logfile_path = lambda x: os.path.join(self.chdir, x) + + if not args: + + pattern = '%s*.log' % self.prefix + args = self.args_to_files([pattern], tail) + + if not args: + if self.chdir: + directory = self.chdir + else: + directory = os.getcwd() + + sys.stderr.write('%s: obj: No arguments specified.\n' % self.name) + sys.stderr.write('%s No %s*.log files found in "%s".\n' % (self.name_spaces, self.prefix, directory)) + sys.stderr.write('%s Type "%s help obj" for help.\n' % (self.name_spaces, self.name)) + sys.exit(1) + + else: + + args = self.args_to_files(args, tail) + + cwd_ = os.getcwd() + os.sep + + if format == 'ascii': + + self.ascii_table(args, tuple(self.stages), self.get_object_counts, logfile_path, object_name) + + elif format == 'gnuplot': + + stage_index = 0 + for s in self.stages: + if stage == s: + break + stage_index = stage_index + 1 + + results = self.collect_results(args, self.get_object_counts, + object_name, stage_index) + + self.gnuplot_results(results) + + else: + + sys.stderr.write('%s: obj: Unknown format "%s".\n' % (self.name, format)) + sys.exit(1) + + return 0 + + # + + def help_run(self): + help = """\ + Usage: scons-time run [OPTIONS] [FILE ...] + + --chdir=DIR Name of unpacked directory for chdir + -f FILE, --file=FILE Read configuration from specified FILE + -h, --help Print this help and exit + -n, --no-exec No execute, just print command lines + --number=NUMBER Put output in files for run NUMBER + --outdir=OUTDIR Put output files in OUTDIR + -p STRING, --prefix=STRING Use STRING as log file/profile prefix + --python=PYTHON Time using the specified PYTHON + -q, --quiet Don't print command lines + --scons=SCONS Time using the specified SCONS + --svn=URL, --subversion=URL Use SCons from Subversion URL + -v, --verbose Display output of commands + """ + sys.stdout.write(self.outdent(help)) + sys.stdout.flush() + + def do_run(self, argv): + """ + """ + run_number_list = [None] + + short_opts = '?f:hnp:qs:v' + + long_opts = [ + 'file=', + 'help', + 'no-exec', + 'number=', + 'outdir=', + 'prefix=', + 'python=', + 'quiet', + 'scons=', + 'svn=', + 'subdir=', + 'subversion=', + 'verbose', + ] + + opts, args = getopt.getopt(argv[1:], short_opts, long_opts) + + for o, a in opts: + if o in ('-f', '--file'): + self.config_file = a + elif o in ('-?', '-h', '--help'): + self.do_help(['help', 'run']) + sys.exit(0) + elif o in ('-n', '--no-exec'): + self.execute = self._do_not_execute + elif o in ('--number',): + run_number_list = self.split_run_numbers(a) + elif o in ('--outdir',): + self.outdir = a + elif o in ('-p', '--prefix'): + self.prefix = a + elif o in ('--python',): + self.python = a + elif o in ('-q', '--quiet'): + self.display = self._do_not_display + elif o in ('-s', '--subdir'): + self.subdir = a + elif o in ('--scons',): + self.scons = a + elif o in ('--svn', '--subversion'): + self.subversion_url = a + elif o in ('-v', '--verbose'): + self.redirect = tee_to_file + self.verbose = True + self.svn_co_flag = '' + + if not args and not self.config_file: + sys.stderr.write('%s: run: No arguments or -f config file specified.\n' % self.name) + sys.stderr.write('%s Type "%s help run" for help.\n' % (self.name_spaces, self.name)) + sys.exit(1) + + if self.config_file: + with open(self.config_file, 'r') as f: + config = f.read() + exec(config, self.__dict__) + + if args: + self.archive_list = args + + archive_file_name = os.path.split(self.archive_list[0])[1] + + if not self.subdir: + self.subdir = self.archive_splitext(archive_file_name)[0] + + if not self.prefix: + self.prefix = self.archive_splitext(archive_file_name)[0] + + prepare = None + if self.subversion_url: + prepare = self.prep_subversion_run + + for run_number in run_number_list: + self.individual_run(run_number, self.archive_list, prepare) + + def split_run_numbers(self, s): + result = [] + for n in s.split(','): + try: + x, y = n.split('-') + except ValueError: + result.append(int(n)) + else: + result.extend(list(range(int(x), int(y)+1))) + return result + + def scons_path(self, dir): + return os.path.join(dir, 'src', 'script', 'scons.py') + + def scons_lib_dir_path(self, dir): + return os.path.join(dir, 'src', 'engine') + + def prep_subversion_run(self, commands, removals): + self.svn_tmpdir = tempfile.mkdtemp(prefix=self.name + '-svn-') + removals.append((shutil.rmtree, 'rm -rf %%s', self.svn_tmpdir)) + + self.scons = self.scons_path(self.svn_tmpdir) + self.scons_lib_dir = self.scons_lib_dir_path(self.svn_tmpdir) + + commands.extend([ + '%(svn)s co %(svn_co_flag)s -r %(run_number)s %(subversion_url)s %(svn_tmpdir)s', + ]) + + def individual_run(self, run_number, archive_list, prepare=None): + """ + Performs an individual run of the default SCons invocations. + """ + + commands = [] + removals = [] + + if prepare: + prepare(commands, removals) + + save_scons = self.scons + save_scons_wrapper = self.scons_wrapper + save_scons_lib_dir = self.scons_lib_dir + + if self.outdir is None: + self.outdir = self.orig_cwd + elif not os.path.isabs(self.outdir): + self.outdir = os.path.join(self.orig_cwd, self.outdir) + + if self.scons is None: + self.scons = self.scons_path(self.orig_cwd) + + if self.scons_lib_dir is None: + self.scons_lib_dir = self.scons_lib_dir_path(self.orig_cwd) + + if self.scons_wrapper is None: + self.scons_wrapper = self.scons + + if not run_number: + run_number = self.find_next_run_number(self.outdir, self.prefix) + + self.run_number = str(run_number) + + self.prefix_run = self.prefix + '-%03d' % run_number + + if self.targets0 is None: + self.targets0 = self.startup_targets + if self.targets1 is None: + self.targets1 = self.targets + if self.targets2 is None: + self.targets2 = self.targets + + self.tmpdir = tempfile.mkdtemp(prefix=self.name + '-') + + commands.extend([ + (os.chdir, 'cd %%s', self.tmpdir), + ]) + + for archive in archive_list: + if not os.path.isabs(archive): + archive = os.path.join(self.orig_cwd, archive) + if os.path.isdir(archive): + dest = os.path.split(archive)[1] + commands.append((shutil.copytree, 'cp -r %%s %%s', archive, dest)) + else: + suffix = self.archive_splitext(archive)[1] + unpack_command = self.unpack_map.get(suffix) + if not unpack_command: + dest = os.path.split(archive)[1] + commands.append((shutil.copyfile, 'cp %%s %%s', archive, dest)) + else: + commands.append(unpack_command + (archive,)) + + commands.extend([ + (os.chdir, 'cd %%s', self.subdir), + ]) + + commands.extend(self.initial_commands) + + commands.extend([ + (lambda: read_tree('.'), + 'find * -type f | xargs cat > /dev/null'), + + (self.set_env, 'export %%s=%%s', + 'SCONS_LIB_DIR', self.scons_lib_dir), + + '%(python)s %(scons_wrapper)s --version', + ]) + + index = 0 + for run_command in self.run_commands: + setattr(self, 'prof%d' % index, self.profile_name(index)) + c = ( + self.log_execute, + self.log_display, + run_command, + self.logfile_name(index), + ) + commands.append(c) + index = index + 1 + + commands.extend([ + (os.chdir, 'cd %%s', self.orig_cwd), + ]) + + if not os.environ.get('PRESERVE'): + commands.extend(removals) + commands.append((shutil.rmtree, 'rm -rf %%s', self.tmpdir)) + + self.run_command_list(commands, self.__dict__) + + self.scons = save_scons + self.scons_lib_dir = save_scons_lib_dir + self.scons_wrapper = save_scons_wrapper + + # + + def help_time(self): + help = """\ + Usage: scons-time time [OPTIONS] FILE [...] + + -C DIR, --chdir=DIR Change to DIR before looking for files + -f FILE, --file=FILE Read configuration from specified FILE + --fmt=FORMAT, --format=FORMAT Print data in specified FORMAT + -h, --help Print this help and exit + -p STRING, --prefix=STRING Use STRING as log file/profile prefix + -t NUMBER, --tail=NUMBER Only report the last NUMBER files + --which=TIMER Plot timings for TIMER: total, + SConscripts, SCons, commands. + """ + sys.stdout.write(self.outdent(help)) + sys.stdout.flush() + + def do_time(self, argv): + + format = 'ascii' + logfile_path = lambda x: x + tail = None + which = 'total' + + short_opts = '?C:f:hp:t:' + + long_opts = [ + 'chdir=', + 'file=', + 'fmt=', + 'format=', + 'help', + 'prefix=', + 'tail=', + 'title=', + 'which=', + ] + + opts, args = getopt.getopt(argv[1:], short_opts, long_opts) + + for o, a in opts: + if o in ('-C', '--chdir'): + self.chdir = a + elif o in ('-f', '--file'): + self.config_file = a + elif o in ('--fmt', '--format'): + format = a + elif o in ('-?', '-h', '--help'): + self.do_help(['help', 'time']) + sys.exit(0) + elif o in ('-p', '--prefix'): + self.prefix = a + elif o in ('-t', '--tail'): + tail = int(a) + elif o in ('--title',): + self.title = a + elif o in ('--which',): + if a not in list(self.time_strings.keys()): + sys.stderr.write('%s: time: Unrecognized timer "%s".\n' % (self.name, a)) + sys.stderr.write('%s Type "%s help time" for help.\n' % (self.name_spaces, self.name)) + sys.exit(1) + which = a + + if self.config_file: + with open(self.config_file, 'r') as f: + config = f.read() + HACK_for_exec(config, self.__dict__) + + if self.chdir: + os.chdir(self.chdir) + logfile_path = lambda x: os.path.join(self.chdir, x) + + if not args: + + pattern = '%s*.log' % self.prefix + args = self.args_to_files([pattern], tail) + + if not args: + if self.chdir: + directory = self.chdir + else: + directory = os.getcwd() + + sys.stderr.write('%s: time: No arguments specified.\n' % self.name) + sys.stderr.write('%s No %s*.log files found in "%s".\n' % (self.name_spaces, self.prefix, directory)) + sys.stderr.write('%s Type "%s help time" for help.\n' % (self.name_spaces, self.name)) + sys.exit(1) + + else: + + args = self.args_to_files(args, tail) + + cwd_ = os.getcwd() + os.sep + + if format == 'ascii': + + columns = ("Total", "SConscripts", "SCons", "commands") + self.ascii_table(args, columns, self.get_debug_times, logfile_path) + + elif format == 'gnuplot': + + results = self.collect_results(args, self.get_debug_times, + self.time_strings[which]) + + self.gnuplot_results(results, fmt='%s %.6f') + + else: + + sys.stderr.write('%s: time: Unknown format "%s".\n' % (self.name, format)) + sys.exit(1) + +if __name__ == '__main__': + opts, args = getopt.getopt(sys.argv[1:], 'h?V', ['help', 'version']) + + ST = SConsTimer() + + for o, a in opts: + if o in ('-?', '-h', '--help'): + ST.do_help(['help']) + sys.exit(0) + elif o in ('-V', '--version'): + sys.stdout.write('scons-time version\n') + sys.exit(0) + + if not args: + sys.stderr.write('Type "%s help" for usage.\n' % ST.name) + sys.exit(1) + + ST.execute_subcommand(args) + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/scripts/scons.bat b/scripts/scons.bat new file mode 100644 index 0000000..10b8637 --- /dev/null +++ b/scripts/scons.bat @@ -0,0 +1,38 @@ +@REM __COPYRIGHT__ +@REM __FILE__ __REVISION__ __DATE__ __DEVELOPER__ +@echo off +set SCONS_ERRORLEVEL= +if "%OS%" == "Windows_NT" goto WinNT + +@REM for 9x/Me you better not have more than 9 args +python -c "from os.path import join; import sys; sys.path = [ join(sys.prefix, 'Lib', 'site-packages', 'scons-__VERSION__'), join(sys.prefix, 'Lib', 'site-packages', 'scons'), join(sys.prefix, 'scons-__VERSION__'), join(sys.prefix, 'scons')] + sys.path; import SCons.Script; SCons.Script.main()" %1 %2 %3 %4 %5 %6 %7 %8 %9 +@REM no way to set exit status of this script for 9x/Me +goto endscons + +@REM Credit where credit is due: we return the exit code despite our +@REM use of setlocal+endlocal using a technique from Bear's Journal: +@REM http://code-bear.com/bearlog/2007/06/01/getting-the-exit-code-from-a-batch-file-that-is-run-from-a-python-program/ + +:WinNT +setlocal +@REM ensure the script will be executed with the Python it was installed for +pushd %~dp0.. +set path=%~dp0;%CD%;%path% +popd +@REM try the script named as the .bat file in current dir, then in Scripts subdir +set scriptname=%~dp0%~n0.py +if not exist "%scriptname%" set scriptname=%~dp0Scripts\%~n0.py +@REM Handle when running from wheel where the script has no .py extension +if not exist "%scriptname%" set scriptname=%~dp0%~n0 +python "%scriptname%" %* +endlocal & set SCONS_ERRORLEVEL=%ERRORLEVEL% + +if NOT "%COMSPEC%" == "%SystemRoot%\system32\cmd.exe" goto returncode +if errorlevel 9009 echo you do not have python in your PATH +goto endscons + +:returncode +exit /B %SCONS_ERRORLEVEL% + +:endscons +call :returncode %SCONS_ERRORLEVEL% diff --git a/scripts/scons.py b/scripts/scons.py new file mode 100755 index 0000000..1dc6c78 --- /dev/null +++ b/scripts/scons.py @@ -0,0 +1,102 @@ +#! /usr/bin/env python +# +# SCons - a Software Constructor +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +__version__ = "__VERSION__" + +__build__ = "__BUILD__" + +__buildsys__ = "__BUILDSYS__" + +__date__ = "__DATE__" + +__developer__ = "__DEVELOPER__" + + +import os +import sys + + +# Python compatibility check +if sys.version_info < (3, 5, 0): + msg = "scons: *** SCons version %s does not run under Python version %s.\n\ +Python >= 3.5 is required.\n" + sys.stderr.write(msg % (__version__, sys.version.split()[0])) + sys.exit(1) + +# Strip the script directory from sys.path so on case-insensitive +# (WIN32) systems Python doesn't think that the "scons" script is the +# "SCons" package. +script_dir = os.path.dirname(os.path.realpath(__file__)) +script_path = os.path.realpath(os.path.dirname(__file__)) +if script_path in sys.path: + sys.path.remove(script_path) + +libs = [] + +if "SCONS_LIB_DIR" in os.environ: + libs.append(os.environ["SCONS_LIB_DIR"]) + +# running from source takes 2nd priority (since 2.3.2), following SCONS_LIB_DIR +source_path = os.path.join(script_path, os.pardir, 'src', 'engine') +if os.path.isdir(source_path): + libs.append(source_path) + +# add local-install locations +local_version = 'scons-local-' + __version__ +local = 'scons-local' +if script_dir: + local_version = os.path.join(script_dir, local_version) + local = os.path.join(script_dir, local) +if os.path.isdir(local_version): + libs.append(os.path.abspath(local_version)) +if os.path.isdir(local): + libs.append(os.path.abspath(local)) + +sys.path = libs + sys.path + +############################################################################## +# END STANDARD SCons SCRIPT HEADER +############################################################################## + +if __name__ == "__main__": + try: + import SCons.Script + except ImportError: + sys.stderr.write("SCons import failed. Unable to find engine files in:\n") + for path in libs: + sys.stderr.write(" {}\n".format(path)) + raise + + # this does all the work, and calls sys.exit + # with the proper exit status when done. + SCons.Script.main() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/scripts/sconsign.py b/scripts/sconsign.py new file mode 100644 index 0000000..726838c --- /dev/null +++ b/scripts/sconsign.py @@ -0,0 +1,652 @@ +#! /usr/bin/env python +# +# SCons - a Software Constructor +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +__version__ = "__VERSION__" + +__build__ = "__BUILD__" + +__buildsys__ = "__BUILDSYS__" + +__date__ = "__DATE__" + +__developer__ = "__DEVELOPER__" + +import os +import sys + +############################################################################## +# BEGIN STANDARD SCons SCRIPT HEADER +# +# This is the cut-and-paste logic so that a self-contained script can +# interoperate correctly with different SCons versions and installation +# locations for the engine. If you modify anything in this section, you +# should also change other scripts that use this same header. +############################################################################## + +# compatibility check +if sys.version_info < (3,5,0): + msg = "scons: *** SCons version %s does not run under Python version %s.\n\ +Python >= 3.5 is required.\n" + sys.stderr.write(msg % (__version__, sys.version.split()[0])) + sys.exit(1) + +# Strip the script directory from sys.path so on case-insensitive +# (WIN32) systems Python doesn't think that the "scons" script is the +# "SCons" package. +script_dir = os.path.dirname(os.path.realpath(__file__)) +script_path = os.path.realpath(os.path.dirname(__file__)) +if script_path in sys.path: + sys.path.remove(script_path) + +libs = [] + +if "SCONS_LIB_DIR" in os.environ: + libs.append(os.environ["SCONS_LIB_DIR"]) + +# running from source takes 2nd priority (since 2.3.2), following SCONS_LIB_DIR +source_path = os.path.join(script_path, os.pardir, 'engine') +if os.path.isdir(source_path): + libs.append(source_path) + +# add local-install locations +local_version = 'scons-local-' + __version__ +local = 'scons-local' +if script_dir: + local_version = os.path.join(script_dir, local_version) + local = os.path.join(script_dir, local) +if os.path.isdir(local_version): + libs.append(os.path.abspath(local_version)) +if os.path.isdir(local): + libs.append(os.path.abspath(local)) + +scons_version = 'scons-%s' % __version__ + +# preferred order of scons lookup paths +prefs = [] + +# if we can find package information, use it +try: + import pkg_resources +except ImportError: + pass +else: + try: + d = pkg_resources.get_distribution('scons') + except pkg_resources.DistributionNotFound: + pass + else: + prefs.append(d.location) + +if sys.platform == 'win32': + # Use only sys.prefix on Windows + prefs.append(sys.prefix) + prefs.append(os.path.join(sys.prefix, 'Lib', 'site-packages')) +else: + # On other (POSIX) platforms, things are more complicated due to + # the variety of path names and library locations. + # Build up some possibilities, then transform them into candidates + temp = [] + if script_dir == 'bin': + # script_dir is `pwd`/bin; + # check `pwd`/lib/scons*. + temp.append(os.getcwd()) + else: + if script_dir in ('.', ''): + script_dir = os.getcwd() + head, tail = os.path.split(script_dir) + if tail == "bin": + # script_dir is /foo/bin; + # check /foo/lib/scons*. + temp.append(head) + + head, tail = os.path.split(sys.prefix) + if tail == "usr": + # sys.prefix is /foo/usr; + # check /foo/usr/lib/scons* first, + # then /foo/usr/local/lib/scons*. + temp.append(sys.prefix) + temp.append(os.path.join(sys.prefix, "local")) + elif tail == "local": + h, t = os.path.split(head) + if t == "usr": + # sys.prefix is /foo/usr/local; + # check /foo/usr/local/lib/scons* first, + # then /foo/usr/lib/scons*. + temp.append(sys.prefix) + temp.append(head) + else: + # sys.prefix is /foo/local; + # check only /foo/local/lib/scons*. + temp.append(sys.prefix) + else: + # sys.prefix is /foo (ends in neither /usr or /local); + # check only /foo/lib/scons*. + temp.append(sys.prefix) + + # suffix these to add to our original prefs: + prefs.extend([os.path.join(x, 'lib') for x in temp]) + prefs.extend([os.path.join(x, 'lib', 'python' + sys.version[:3], + 'site-packages') for x in temp]) + + + # Add the parent directory of the current python's library to the + # preferences. This picks up differences between, e.g., lib and lib64, + # and finds the base location in case of a non-copying virtualenv. + try: + libpath = os.__file__ + except AttributeError: + pass + else: + # Split /usr/libfoo/python*/os.py to /usr/libfoo/python*. + libpath, _ = os.path.split(libpath) + # Split /usr/libfoo/python* to /usr/libfoo + libpath, tail = os.path.split(libpath) + # Check /usr/libfoo/scons*. + prefs.append(libpath) + +# Look first for 'scons-__version__' in all of our preference libs, +# then for 'scons'. Skip paths that do not exist. +libs.extend([os.path.join(x, scons_version) for x in prefs if os.path.isdir(x)]) +libs.extend([os.path.join(x, 'scons') for x in prefs if os.path.isdir(x)]) + +sys.path = libs + sys.path + +############################################################################## +# END STANDARD SCons SCRIPT HEADER +############################################################################## + +import SCons.compat + +try: + import whichdb + + whichdb = whichdb.whichdb +except ImportError as e: + from dbm import whichdb + +import time +import pickle + +import SCons.SConsign + + +def my_whichdb(filename): + if filename[-7:] == ".dblite": + return "SCons.dblite" + try: + with open(filename + ".dblite", "rb"): + return "SCons.dblite" + except IOError: + pass + return _orig_whichdb(filename) + + +# Should work on python2 +_orig_whichdb = whichdb +whichdb = my_whichdb + +# was changed for python3 +#_orig_whichdb = whichdb.whichdb +#dbm.whichdb = my_whichdb + +def my_import(mname): + import imp + + if '.' in mname: + i = mname.rfind('.') + parent = my_import(mname[:i]) + fp, pathname, description = imp.find_module(mname[i+1:], + parent.__path__) + else: + fp, pathname, description = imp.find_module(mname) + return imp.load_module(mname, fp, pathname, description) + + +class Flagger(object): + default_value = 1 + + def __setitem__(self, item, value): + self.__dict__[item] = value + self.default_value = 0 + + def __getitem__(self, item): + return self.__dict__.get(item, self.default_value) + + +Do_Call = None +Print_Directories = [] +Print_Entries = [] +Print_Flags = Flagger() +Verbose = 0 +Readable = 0 +Warns = 0 + + +def default_mapper(entry, name): + """ + Stringify an entry that doesn't have an explicit mapping. + + Args: + entry: entry + name: field name + + Returns: str + + """ + try: + val = eval("entry." + name) + except AttributeError: + val = None + if sys.version_info.major >= 3 and isinstance(val, bytes): + # This is a dirty hack for py 2/3 compatibility. csig is a bytes object + # in Python3 while Python2 bytes are str. Hence, we decode the csig to a + # Python3 string + val = val.decode() + return str(val) + + +def map_action(entry, _): + """ + Stringify an action entry and signature. + + Args: + entry: action entry + second argument is not used + + Returns: str + + """ + try: + bact = entry.bact + bactsig = entry.bactsig + except AttributeError: + return None + return '%s [%s]' % (bactsig, bact) + + +def map_timestamp(entry, _): + """ + Stringify a timestamp entry. + + Args: + entry: timestamp entry + second argument is not used + + Returns: str + + """ + try: + timestamp = entry.timestamp + except AttributeError: + timestamp = None + if Readable and timestamp: + return "'" + time.ctime(timestamp) + "'" + else: + return str(timestamp) + + +def map_bkids(entry, _): + """ + Stringify an implicit entry. + + Args: + entry: + second argument is not used + + Returns: str + + """ + try: + bkids = entry.bsources + entry.bdepends + entry.bimplicit + bkidsigs = entry.bsourcesigs + entry.bdependsigs + entry.bimplicitsigs + except AttributeError: + return None + + if len(bkids) != len(bkidsigs): + global Warns + Warns += 1 + # add warning to result rather than direct print so it will line up + msg = "Warning: missing information, {} ids but {} sigs" + result = [msg.format(len(bkids), len(bkidsigs))] + else: + result = [] + result += [nodeinfo_string(bkid, bkidsig, " ") + for bkid, bkidsig in zip(bkids, bkidsigs)] + if not result: + return None + return "\n ".join(result) + + +map_field = { + 'action' : map_action, + 'timestamp' : map_timestamp, + 'bkids' : map_bkids, +} + +map_name = { + 'implicit' : 'bkids', +} + + +def field(name, entry, verbose=Verbose): + if not Print_Flags[name]: + return None + fieldname = map_name.get(name, name) + mapper = map_field.get(fieldname, default_mapper) + val = mapper(entry, name) + if verbose: + val = name + ": " + val + return val + + +def nodeinfo_raw(name, ninfo, prefix=""): + # This just formats the dictionary, which we would normally use str() + # to do, except that we want the keys sorted for deterministic output. + d = ninfo.__getstate__() + try: + keys = ninfo.field_list + ['_version_id'] + except AttributeError: + keys = sorted(d.keys()) + l = [] + for k in keys: + l.append('%s: %s' % (repr(k), repr(d.get(k)))) + if '\n' in name: + name = repr(name) + return name + ': {' + ', '.join(l) + '}' + + +def nodeinfo_cooked(name, ninfo, prefix=""): + try: + field_list = ninfo.field_list + except AttributeError: + field_list = [] + if '\n' in name: + name = repr(name) + outlist = [name + ':'] + [ + f for f in [field(x, ninfo, Verbose) for x in field_list] if f + ] + if Verbose: + sep = '\n ' + prefix + else: + sep = ' ' + return sep.join(outlist) + + +nodeinfo_string = nodeinfo_cooked + + +def printfield(name, entry, prefix=""): + outlist = field("implicit", entry, 0) + if outlist: + if Verbose: + print(" implicit:") + print(" " + outlist) + outact = field("action", entry, 0) + if outact: + if Verbose: + print(" action: " + outact) + else: + print(" " + outact) + + +def printentries(entries, location): + if Print_Entries: + for name in Print_Entries: + try: + entry = entries[name] + except KeyError: + err = "sconsign: no entry `%s' in `%s'\n" % (name, location) + sys.stderr.write(err) + else: + try: + ninfo = entry.ninfo + except AttributeError: + print(name + ":") + else: + print(nodeinfo_string(name, entry.ninfo)) + printfield(name, entry.binfo) + else: + for name in sorted(entries.keys()): + entry = entries[name] + try: + ninfo = entry.ninfo + except AttributeError: + print(name + ":") + else: + print(nodeinfo_string(name, entry.ninfo)) + printfield(name, entry.binfo) + + +class Do_SConsignDB(object): + def __init__(self, dbm_name, dbm): + self.dbm_name = dbm_name + self.dbm = dbm + + def __call__(self, fname): + # The *dbm modules stick their own file suffixes on the names + # that are passed in. This causes us to jump through some + # hoops here. + try: + # Try opening the specified file name. Example: + # SPECIFIED OPENED BY self.dbm.open() + # --------- ------------------------- + # .sconsign => .sconsign.dblite + # .sconsign.dblite => .sconsign.dblite.dblite + db = self.dbm.open(fname, "r") + except (IOError, OSError) as e: + print_e = e + try: + # That didn't work, so try opening the base name, + # so that if they actually passed in 'sconsign.dblite' + # (for example), the dbm module will put the suffix back + # on for us and open it anyway. + db = self.dbm.open(os.path.splitext(fname)[0], "r") + except (IOError, OSError): + # That didn't work either. See if the file name + # they specified even exists (independent of the dbm + # suffix-mangling). + try: + with open(fname, "rb"): + pass # this is a touch only, we don't use it here. + except (IOError, OSError) as e: + # Nope, that file doesn't even exist, so report that + # fact back. + print_e = e + sys.stderr.write("sconsign: %s\n" % print_e) + return + except KeyboardInterrupt: + raise + except pickle.UnpicklingError: + sys.stderr.write("sconsign: ignoring invalid `%s' file `%s'\n" + % (self.dbm_name, fname)) + return + except Exception as e: + sys.stderr.write("sconsign: ignoring invalid `%s' file `%s': %s\n" + % (self.dbm_name, fname, e)) + exc_type, _, _ = sys.exc_info() + if exc_type.__name__ == "ValueError" and sys.version_info < (3,0,0): + sys.stderr.write("Python 2 only supports pickle protocols 0-2.\n") + return + + if Print_Directories: + for dir in Print_Directories: + try: + val = db[dir] + except KeyError: + err = "sconsign: no dir `%s' in `%s'\n" % (dir, args[0]) + sys.stderr.write(err) + else: + self.printentries(dir, val) + else: + for dir in sorted(db.keys()): + self.printentries(dir, db[dir]) + + @staticmethod + def printentries(dir, val): + try: + print('=== ' + dir + ':') + except TypeError: + print('=== ' + dir.decode() + ':') + printentries(pickle.loads(val), dir) + + +def Do_SConsignDir(name): + try: + with open(name, 'rb') as fp: + try: + sconsign = SCons.SConsign.Dir(fp) + except KeyboardInterrupt: + raise + except pickle.UnpicklingError: + err = "sconsign: ignoring invalid .sconsign file `%s'\n" % name + sys.stderr.write(err) + return + except Exception as e: + err = "sconsign: ignoring invalid .sconsign file `%s': %s\n" % (name, e) + sys.stderr.write(err) + return + printentries(sconsign.entries, args[0]) + except (IOError, OSError) as e: + sys.stderr.write("sconsign: %s\n" % e) + return + + +############################################################################## + +import getopt + +helpstr = """\ +Usage: sconsign [OPTIONS] [FILE ...] +Options: + -a, --act, --action Print build action information. + -c, --csig Print content signature information. + -d DIR, --dir=DIR Print only info about DIR. + -e ENTRY, --entry=ENTRY Print only info about ENTRY. + -f FORMAT, --format=FORMAT FILE is in the specified FORMAT. + -h, --help Print this message and exit. + -i, --implicit Print implicit dependency information. + -r, --readable Print timestamps in human-readable form. + --raw Print raw Python object representations. + -s, --size Print file sizes. + -t, --timestamp Print timestamp information. + -v, --verbose Verbose, describe each field. +""" + +try: + opts, args = getopt.getopt(sys.argv[1:], "acd:e:f:hirstv", + ['act', 'action', + 'csig', 'dir=', 'entry=', + 'format=', 'help', 'implicit', + 'raw', 'readable', + 'size', 'timestamp', 'verbose']) +except getopt.GetoptError as err: + sys.stderr.write(str(err) + '\n') + print(helpstr) + sys.exit(2) + +for o, a in opts: + if o in ('-a', '--act', '--action'): + Print_Flags['action'] = 1 + elif o in ('-c', '--csig'): + Print_Flags['csig'] = 1 + elif o in ('-d', '--dir'): + Print_Directories.append(a) + elif o in ('-e', '--entry'): + Print_Entries.append(a) + elif o in ('-f', '--format'): + # Try to map the given DB format to a known module + # name, that we can then try to import... + Module_Map = {'dblite': 'SCons.dblite', 'sconsign': None} + dbm_name = Module_Map.get(a, a) + if dbm_name: + try: + if dbm_name != "SCons.dblite": + dbm = my_import(dbm_name) + else: + import SCons.dblite + + dbm = SCons.dblite + # Ensure that we don't ignore corrupt DB files, + # this was handled by calling my_import('SCons.dblite') + # again in earlier versions... + SCons.dblite.ignore_corrupt_dbfiles = 0 + except ImportError: + sys.stderr.write("sconsign: illegal file format `%s'\n" % a) + print(helpstr) + sys.exit(2) + Do_Call = Do_SConsignDB(a, dbm) + else: + Do_Call = Do_SConsignDir + elif o in ('-h', '--help'): + print(helpstr) + sys.exit(0) + elif o in ('-i', '--implicit'): + Print_Flags['implicit'] = 1 + elif o in ('--raw',): + nodeinfo_string = nodeinfo_raw + elif o in ('-r', '--readable'): + Readable = 1 + elif o in ('-s', '--size'): + Print_Flags['size'] = 1 + elif o in ('-t', '--timestamp'): + Print_Flags['timestamp'] = 1 + elif o in ('-v', '--verbose'): + Verbose = 1 + +if Do_Call: + for a in args: + Do_Call(a) +else: + if not args: + args = [".sconsign.dblite"] + for a in args: + dbm_name = whichdb(a) + if dbm_name: + Map_Module = {'SCons.dblite': 'dblite'} + if dbm_name != "SCons.dblite": + dbm = my_import(dbm_name) + else: + import SCons.dblite + + dbm = SCons.dblite + # Ensure that we don't ignore corrupt DB files, + # this was handled by calling my_import('SCons.dblite') + # again in earlier versions... + SCons.dblite.ignore_corrupt_dbfiles = 0 + Do_SConsignDB(Map_Module.get(dbm_name, dbm_name), dbm)(a) + else: + Do_SConsignDir(a) + + if Warns: + print("NOTE: there were %d warnings, please check output" % Warns) +sys.exit(0) + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/src/script/MANIFEST.in b/src/script/MANIFEST.in deleted file mode 100644 index d10cc82..0000000 --- a/src/script/MANIFEST.in +++ /dev/null @@ -1,4 +0,0 @@ -scons -sconsign -scons-time -scons-configure-cache diff --git a/src/script/scons-configure-cache.py b/src/script/scons-configure-cache.py deleted file mode 100644 index 716315c..0000000 --- a/src/script/scons-configure-cache.py +++ /dev/null @@ -1,177 +0,0 @@ -#! /usr/bin/env python -# -# SCons - a Software Constructor -# -# __COPYRIGHT__ -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -"""Show or convert the configuration of an SCons cache directory. - -A cache of derived files is stored by file signature. -The files are split into directories named by the first few -digits of the signature. The prefix length used for directory -names can be changed by this script. -""" - -import argparse -import glob -import json -import os - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - -__version__ = "__VERSION__" - -__build__ = "__BUILD__" - -__buildsys__ = "__BUILDSYS__" - -__date__ = "__DATE__" - -__developer__ = "__DEVELOPER__" - - -def rearrange_cache_entries(current_prefix_len, new_prefix_len): - """Move cache files if prefix length changed. - - Move the existing cache files to new directories of the - appropriate name length and clean up the old directories. - """ - print('Changing prefix length from', current_prefix_len, - 'to', new_prefix_len) - dirs = set() - old_dirs = set() - for file in glob.iglob(os.path.join('*', '*')): - name = os.path.basename(file) - dname = name[:current_prefix_len].upper() - if dname not in old_dirs: - print('Migrating', dname) - old_dirs.add(dname) - dname = name[:new_prefix_len].upper() - if dname not in dirs: - os.mkdir(dname) - dirs.add(dname) - os.rename(file, os.path.join(dname, name)) - - # Now delete the original directories - for dname in old_dirs: - os.rmdir(dname) - - -# The configuration dictionary should have one entry per entry in the -# cache config. The value of each entry should include the following: -# implicit - (optional) This is to allow adding a new config entry and also -# changing the behaviour of the system at the same time. This -# indicates the value the config entry would have had if it had -# been specified. -# default - The value the config entry should have if it wasn't previously -# specified -# command-line - parameters to pass to ArgumentParser.add_argument -# converter - (optional) Function to call if conversion is required -# if this configuration entry changes -config_entries = { - 'prefix_len': { - 'implicit': 1, - 'default': 2, - 'command-line': { - 'help': 'Length of cache file name used as subdirectory prefix', - 'metavar': '', - 'type': int - }, - 'converter': rearrange_cache_entries - } -} - -parser = argparse.ArgumentParser( - description='Modify the configuration of an scons cache directory', - epilog=''' - Unspecified options will not be changed unless they are not - set at all, in which case they are set to an appropriate default. - ''') - -parser.add_argument('cache-dir', help='Path to scons cache directory') -for param in config_entries: - parser.add_argument('--' + param.replace('_', '-'), - **config_entries[param]['command-line']) -parser.add_argument('--version', - action='version', - version='%(prog)s 1.0') -parser.add_argument('--show', - action="store_true", - help="show current configuration") - -# Get the command line as a dict without any of the unspecified entries. -args = dict([x for x in vars(parser.parse_args()).items() if x[1]]) - -# It seems somewhat strange to me, but positional arguments don't get the - -# in the name changed to _, whereas optional arguments do... -cache = args['cache-dir'] -if not os.path.isdir(cache): - raise RuntimeError("There is no cache directory named %s" % cache) -os.chdir(cache) -del args['cache-dir'] - -if not os.path.exists('config'): - # old config dirs did not have a 'config' file. Try to update. - # Validate the only files in the directory are directories 0-9, a-f - expected = ['{:X}'.format(x) for x in range(0, 16)] - if not set(os.listdir('.')).issubset(expected): - raise RuntimeError( - "%s does not look like a valid version 1 cache directory" % cache) - config = dict() -else: - with open('config') as conf: - config = json.load(conf) - -if args.get('show', None): - print("Current configuration in '%s':" % cache) - print(json.dumps(config, sort_keys=True, - indent=4, separators=(',', ': '))) - # in case of the show argument, emit some stats as well - file_count = 0 - for _, _, files in os.walk('.'): - file_count += len(files) - if file_count: # skip config file if it exists - file_count -= 1 - print("Cache contains %s files" % file_count) - del args['show'] - -# Find any keys that are not currently set but should be -for key in config_entries: - if key not in config: - if 'implicit' in config_entries[key]: - config[key] = config_entries[key]['implicit'] - else: - config[key] = config_entries[key]['default'] - if key not in args: - args[key] = config_entries[key]['default'] - -# Now go through each entry in args to see if it changes an existing config -# setting. -for key in args: - if args[key] != config[key]: - if 'converter' in config_entries[key]: - config_entries[key]['converter'](config[key], args[key]) - config[key] = args[key] - -# and write the updated config file -with open('config', 'w') as conf: - json.dump(config, conf) diff --git a/src/script/scons-time.py b/src/script/scons-time.py deleted file mode 100644 index e4dd863..0000000 --- a/src/script/scons-time.py +++ /dev/null @@ -1,1480 +0,0 @@ -#!/usr/bin/env python -# -# scons-time - run SCons timings and collect statistics -# -# A script for running a configuration through SCons with a standard -# set of invocations to collect timing and memory statistics and to -# capture the results in a consistent set of output files for display -# and analysis. -# - -# -# __COPYRIGHT__ -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - -import getopt -import glob -import os -import re -import shutil -import sys -import tempfile -import time -import subprocess - -def HACK_for_exec(cmd, *args): - """ - For some reason, Python won't allow an exec() within a function - that also declares an internal function (including lambda functions). - This function is a hack that calls exec() in a function with no - internal functions. - """ - if not args: exec(cmd) - elif len(args) == 1: exec(cmd, args[0]) - else: exec(cmd, args[0], args[1]) - -class Plotter(object): - def increment_size(self, largest): - """ - Return the size of each horizontal increment line for a specified - maximum value. This returns a value that will provide somewhere - between 5 and 9 horizontal lines on the graph, on some set of - boundaries that are multiples of 10/100/1000/etc. - """ - i = largest // 5 - if not i: - return largest - multiplier = 1 - while i >= 10: - i = i // 10 - multiplier = multiplier * 10 - return i * multiplier - - def max_graph_value(self, largest): - # Round up to next integer. - largest = int(largest) + 1 - increment = self.increment_size(largest) - return ((largest + increment - 1) // increment) * increment - -class Line(object): - def __init__(self, points, type, title, label, comment, fmt="%s %s"): - self.points = points - self.type = type - self.title = title - self.label = label - self.comment = comment - self.fmt = fmt - - def print_label(self, inx, x, y): - if self.label: - print('set label %s "%s" at %0.1f,%0.1f right' % (inx, self.label, x, y)) - - def plot_string(self): - if self.title: - title_string = 'title "%s"' % self.title - else: - title_string = 'notitle' - return "'-' %s with lines lt %s" % (title_string, self.type) - - def print_points(self, fmt=None): - if fmt is None: - fmt = self.fmt - if self.comment: - print('# %s' % self.comment) - for x, y in self.points: - # If y is None, it usually represents some kind of break - # in the line's index number. We might want to represent - # this some way rather than just drawing the line straight - # between the two points on either side. - if y is not None: - print(fmt % (x, y)) - print('e') - - def get_x_values(self): - return [ p[0] for p in self.points ] - - def get_y_values(self): - return [ p[1] for p in self.points ] - -class Gnuplotter(Plotter): - - def __init__(self, title, key_location): - self.lines = [] - self.title = title - self.key_location = key_location - - def line(self, points, type, title=None, label=None, comment=None, fmt='%s %s'): - if points: - line = Line(points, type, title, label, comment, fmt) - self.lines.append(line) - - def plot_string(self, line): - return line.plot_string() - - def vertical_bar(self, x, type, label, comment): - if self.get_min_x() <= x <= self.get_max_x(): - points = [(x, 0), (x, self.max_graph_value(self.get_max_y()))] - self.line(points, type, label, comment) - - def get_all_x_values(self): - result = [] - for line in self.lines: - result.extend(line.get_x_values()) - return [r for r in result if r is not None] - - def get_all_y_values(self): - result = [] - for line in self.lines: - result.extend(line.get_y_values()) - return [r for r in result if r is not None] - - def get_min_x(self): - try: - return self.min_x - except AttributeError: - try: - self.min_x = min(self.get_all_x_values()) - except ValueError: - self.min_x = 0 - return self.min_x - - def get_max_x(self): - try: - return self.max_x - except AttributeError: - try: - self.max_x = max(self.get_all_x_values()) - except ValueError: - self.max_x = 0 - return self.max_x - - def get_min_y(self): - try: - return self.min_y - except AttributeError: - try: - self.min_y = min(self.get_all_y_values()) - except ValueError: - self.min_y = 0 - return self.min_y - - def get_max_y(self): - try: - return self.max_y - except AttributeError: - try: - self.max_y = max(self.get_all_y_values()) - except ValueError: - self.max_y = 0 - return self.max_y - - def draw(self): - - if not self.lines: - return - - if self.title: - print('set title "%s"' % self.title) - print('set key %s' % self.key_location) - - min_y = self.get_min_y() - max_y = self.max_graph_value(self.get_max_y()) - incr = (max_y - min_y) / 10.0 - start = min_y + (max_y / 2.0) + (2.0 * incr) - position = [ start - (i * incr) for i in range(5) ] - - inx = 1 - for line in self.lines: - line.print_label(inx, line.points[0][0]-1, - position[(inx-1) % len(position)]) - inx += 1 - - plot_strings = [ self.plot_string(l) for l in self.lines ] - print('plot ' + ', \\\n '.join(plot_strings)) - - for line in self.lines: - line.print_points() - - - -def untar(fname): - import tarfile - tar = tarfile.open(name=fname, mode='r') - for tarinfo in tar: - tar.extract(tarinfo) - tar.close() - -def unzip(fname): - import zipfile - zf = zipfile.ZipFile(fname, 'r') - for name in zf.namelist(): - dir = os.path.dirname(name) - try: - os.makedirs(dir) - except: - pass - with open(name, 'wb') as f: - f.write(zf.read(name)) - -def read_tree(dir): - for dirpath, dirnames, filenames in os.walk(dir): - for fn in filenames: - fn = os.path.join(dirpath, fn) - if os.path.isfile(fn): - with open(fn, 'rb') as f: - f.read() - -def redirect_to_file(command, log): - return '%s > %s 2>&1' % (command, log) - -def tee_to_file(command, log): - return '%s 2>&1 | tee %s' % (command, log) - - - -class SConsTimer(object): - """ - Usage: scons-time SUBCOMMAND [ARGUMENTS] - Type "scons-time help SUBCOMMAND" for help on a specific subcommand. - - Available subcommands: - func Extract test-run data for a function - help Provides help - mem Extract --debug=memory data from test runs - obj Extract --debug=count data from test runs - time Extract --debug=time data from test runs - run Runs a test configuration - """ - - name = 'scons-time' - name_spaces = ' '*len(name) - - def makedict(**kw): - return kw - - default_settings = makedict( - chdir = None, - config_file = None, - initial_commands = [], - key_location = 'bottom left', - orig_cwd = os.getcwd(), - outdir = None, - prefix = '', - python = '"%s"' % sys.executable, - redirect = redirect_to_file, - scons = None, - scons_flags = '--debug=count --debug=memory --debug=time --debug=memoizer', - scons_lib_dir = None, - scons_wrapper = None, - startup_targets = '--help', - subdir = None, - subversion_url = None, - svn = 'svn', - svn_co_flag = '-q', - tar = 'tar', - targets = '', - targets0 = None, - targets1 = None, - targets2 = None, - title = None, - unzip = 'unzip', - verbose = False, - vertical_bars = [], - - unpack_map = { - '.tar.gz' : (untar, '%(tar)s xzf %%s'), - '.tgz' : (untar, '%(tar)s xzf %%s'), - '.tar' : (untar, '%(tar)s xf %%s'), - '.zip' : (unzip, '%(unzip)s %%s'), - }, - ) - - run_titles = [ - 'Startup', - 'Full build', - 'Up-to-date build', - ] - - run_commands = [ - '%(python)s %(scons_wrapper)s %(scons_flags)s --profile=%(prof0)s %(targets0)s', - '%(python)s %(scons_wrapper)s %(scons_flags)s --profile=%(prof1)s %(targets1)s', - '%(python)s %(scons_wrapper)s %(scons_flags)s --profile=%(prof2)s %(targets2)s', - ] - - stages = [ - 'pre-read', - 'post-read', - 'pre-build', - 'post-build', - ] - - stage_strings = { - 'pre-read' : 'Memory before reading SConscript files:', - 'post-read' : 'Memory after reading SConscript files:', - 'pre-build' : 'Memory before building targets:', - 'post-build' : 'Memory after building targets:', - } - - memory_string_all = 'Memory ' - - default_stage = stages[-1] - - time_strings = { - 'total' : 'Total build time', - 'SConscripts' : 'Total SConscript file execution time', - 'SCons' : 'Total SCons execution time', - 'commands' : 'Total command execution time', - } - - time_string_all = 'Total .* time' - - # - - def __init__(self): - self.__dict__.update(self.default_settings) - - # Functions for displaying and executing commands. - - def subst(self, x, dictionary): - try: - return x % dictionary - except TypeError: - # x isn't a string (it's probably a Python function), - # so just return it. - return x - - def subst_variables(self, command, dictionary): - """ - Substitutes (via the format operator) the values in the specified - dictionary into the specified command. - - The command can be an (action, string) tuple. In all cases, we - perform substitution on strings and don't worry if something isn't - a string. (It's probably a Python function to be executed.) - """ - try: - command + '' - except TypeError: - action = command[0] - string = command[1] - args = command[2:] - else: - action = command - string = action - args = (()) - action = self.subst(action, dictionary) - string = self.subst(string, dictionary) - return (action, string, args) - - def _do_not_display(self, msg, *args): - pass - - def display(self, msg, *args): - """ - Displays the specified message. - - Each message is prepended with a standard prefix of our name - plus the time. - """ - if callable(msg): - msg = msg(*args) - else: - msg = msg % args - if msg is None: - return - fmt = '%s[%s]: %s\n' - sys.stdout.write(fmt % (self.name, time.strftime('%H:%M:%S'), msg)) - - def _do_not_execute(self, action, *args): - pass - - def execute(self, action, *args): - """ - Executes the specified action. - - The action is called if it's a callable Python function, and - otherwise passed to os.system(). - """ - if callable(action): - action(*args) - else: - os.system(action % args) - - def run_command_list(self, commands, dict): - """ - Executes a list of commands, substituting values from the - specified dictionary. - """ - commands = [ self.subst_variables(c, dict) for c in commands ] - for action, string, args in commands: - self.display(string, *args) - sys.stdout.flush() - status = self.execute(action, *args) - if status: - sys.exit(status) - - def log_display(self, command, log): - command = self.subst(command, self.__dict__) - if log: - command = self.redirect(command, log) - return command - - def log_execute(self, command, log): - command = self.subst(command, self.__dict__) - p = os.popen(command) - output = p.read() - p.close() - #TODO: convert to subrocess, os.popen is obsolete. This didn't work: - #process = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True) - #output = process.stdout.read() - #process.stdout.close() - #process.wait() - if self.verbose: - sys.stdout.write(output) - # TODO: Figure out - # Not sure we need to write binary here - with open(log, 'w') as f: - f.write(str(output)) - - def archive_splitext(self, path): - """ - Splits an archive name into a filename base and extension. - - This is like os.path.splitext() (which it calls) except that it - also looks for '.tar.gz' and treats it as an atomic extensions. - """ - if path.endswith('.tar.gz'): - return path[:-7], path[-7:] - else: - return os.path.splitext(path) - - def args_to_files(self, args, tail=None): - """ - Takes a list of arguments, expands any glob patterns, and - returns the last "tail" files from the list. - """ - files = [] - for a in args: - files.extend(sorted(glob.glob(a))) - - if tail: - files = files[-tail:] - - return files - - def ascii_table(self, files, columns, - line_function, file_function=lambda x: x, - *args, **kw): - - header_fmt = ' '.join(['%12s'] * len(columns)) - line_fmt = header_fmt + ' %s' - - print(header_fmt % columns) - - for file in files: - t = line_function(file, *args, **kw) - if t is None: - t = [] - diff = len(columns) - len(t) - if diff > 0: - t += [''] * diff - t.append(file_function(file)) - print(line_fmt % tuple(t)) - - def collect_results(self, files, function, *args, **kw): - results = {} - - for file in files: - base = os.path.splitext(file)[0] - run, index = base.split('-')[-2:] - - run = int(run) - index = int(index) - - value = function(file, *args, **kw) - - try: - r = results[index] - except KeyError: - r = [] - results[index] = r - r.append((run, value)) - - return results - - def doc_to_help(self, obj): - """ - Translates an object's __doc__ string into help text. - - This strips a consistent number of spaces from each line in the - help text, essentially "outdenting" the text to the left-most - column. - """ - doc = obj.__doc__ - if doc is None: - return '' - return self.outdent(doc) - - def find_next_run_number(self, dir, prefix): - """ - Returns the next run number in a directory for the specified prefix. - - Examines the contents the specified directory for files with the - specified prefix, extracts the run numbers from each file name, - and returns the next run number after the largest it finds. - """ - x = re.compile(re.escape(prefix) + '-([0-9]+).*') - matches = [x.match(e) for e in os.listdir(dir)] - matches = [_f for _f in matches if _f] - if not matches: - return 0 - run_numbers = [int(m.group(1)) for m in matches] - return int(max(run_numbers)) + 1 - - def gnuplot_results(self, results, fmt='%s %.3f'): - """ - Prints out a set of results in Gnuplot format. - """ - gp = Gnuplotter(self.title, self.key_location) - - for i in sorted(results.keys()): - try: - t = self.run_titles[i] - except IndexError: - t = '??? %s ???' % i - results[i].sort() - gp.line(results[i], i+1, t, None, t, fmt=fmt) - - for bar_tuple in self.vertical_bars: - try: - x, type, label, comment = bar_tuple - except ValueError: - x, type, label = bar_tuple - comment = label - gp.vertical_bar(x, type, label, comment) - - gp.draw() - - def logfile_name(self, invocation): - """ - Returns the absolute path of a log file for the specificed - invocation number. - """ - name = self.prefix_run + '-%d.log' % invocation - return os.path.join(self.outdir, name) - - def outdent(self, s): - """ - Strip as many spaces from each line as are found at the beginning - of the first line in the list. - """ - lines = s.split('\n') - if lines[0] == '': - lines = lines[1:] - spaces = re.match(' *', lines[0]).group(0) - def strip_initial_spaces(l, s=spaces): - if l.startswith(spaces): - l = l[len(spaces):] - return l - return '\n'.join([ strip_initial_spaces(l) for l in lines ]) + '\n' - - def profile_name(self, invocation): - """ - Returns the absolute path of a profile file for the specified - invocation number. - """ - name = self.prefix_run + '-%d.prof' % invocation - return os.path.join(self.outdir, name) - - def set_env(self, key, value): - os.environ[key] = value - - # - - def get_debug_times(self, file, time_string=None): - """ - Fetch times from the --debug=time strings in the specified file. - """ - if time_string is None: - search_string = self.time_string_all - else: - search_string = time_string - with open(file) as f: - contents = f.read() - if not contents: - sys.stderr.write('file %s has no contents!\n' % repr(file)) - return None - result = re.findall(r'%s: ([\d.]*)' % search_string, contents)[-4:] - result = [ float(r) for r in result ] - if time_string is not None: - try: - result = result[0] - except IndexError: - sys.stderr.write('file %s has no results!\n' % repr(file)) - return None - return result - - def get_function_profile(self, file, function): - """ - Returns the file, line number, function name, and cumulative time. - """ - try: - import pstats - except ImportError as e: - sys.stderr.write('%s: func: %s\n' % (self.name, e)) - sys.stderr.write('%s This version of Python is missing the profiler.\n' % self.name_spaces) - sys.stderr.write('%s Cannot use the "func" subcommand.\n' % self.name_spaces) - sys.exit(1) - statistics = pstats.Stats(file).stats - matches = [ e for e in statistics.items() if e[0][2] == function ] - r = matches[0] - return r[0][0], r[0][1], r[0][2], r[1][3] - - def get_function_time(self, file, function): - """ - Returns just the cumulative time for the specified function. - """ - return self.get_function_profile(file, function)[3] - - def get_memory(self, file, memory_string=None): - """ - Returns a list of integers of the amount of memory used. The - default behavior is to return all the stages. - """ - if memory_string is None: - search_string = self.memory_string_all - else: - search_string = memory_string - with open(file) as f: - lines = f.readlines() - lines = [ l for l in lines if l.startswith(search_string) ][-4:] - result = [ int(l.split()[-1]) for l in lines[-4:] ] - if len(result) == 1: - result = result[0] - return result - - def get_object_counts(self, file, object_name, index=None): - """ - Returns the counts of the specified object_name. - """ - object_string = ' ' + object_name + '\n' - with open(file) as f: - lines = f.readlines() - line = [ l for l in lines if l.endswith(object_string) ][0] - result = [ int(field) for field in line.split()[:4] ] - if index is not None: - result = result[index] - return result - - - command_alias = {} - - def execute_subcommand(self, argv): - """ - Executes the do_*() function for the specified subcommand (argv[0]). - """ - if not argv: - return - cmdName = self.command_alias.get(argv[0], argv[0]) - try: - func = getattr(self, 'do_' + cmdName) - except AttributeError: - return self.default(argv) - try: - return func(argv) - except TypeError as e: - sys.stderr.write("%s %s: %s\n" % (self.name, cmdName, e)) - import traceback - traceback.print_exc(file=sys.stderr) - sys.stderr.write("Try '%s help %s'\n" % (self.name, cmdName)) - - def default(self, argv): - """ - The default behavior for an unknown subcommand. Prints an - error message and exits. - """ - sys.stderr.write('%s: Unknown subcommand "%s".\n' % (self.name, argv[0])) - sys.stderr.write('Type "%s help" for usage.\n' % self.name) - sys.exit(1) - - # - - def do_help(self, argv): - """ - """ - if argv[1:]: - for arg in argv[1:]: - try: - func = getattr(self, 'do_' + arg) - except AttributeError: - sys.stderr.write('%s: No help for "%s"\n' % (self.name, arg)) - else: - try: - help = getattr(self, 'help_' + arg) - except AttributeError: - sys.stdout.write(self.doc_to_help(func)) - sys.stdout.flush() - else: - help() - else: - doc = self.doc_to_help(self.__class__) - if doc: - sys.stdout.write(doc) - sys.stdout.flush() - return None - - # - - def help_func(self): - help = """\ - Usage: scons-time func [OPTIONS] FILE [...] - - -C DIR, --chdir=DIR Change to DIR before looking for files - -f FILE, --file=FILE Read configuration from specified FILE - --fmt=FORMAT, --format=FORMAT Print data in specified FORMAT - --func=NAME, --function=NAME Report time for function NAME - -h, --help Print this help and exit - -p STRING, --prefix=STRING Use STRING as log file/profile prefix - -t NUMBER, --tail=NUMBER Only report the last NUMBER files - --title=TITLE Specify the output plot TITLE - """ - sys.stdout.write(self.outdent(help)) - sys.stdout.flush() - - def do_func(self, argv): - """ - """ - format = 'ascii' - function_name = '_main' - tail = None - - short_opts = '?C:f:hp:t:' - - long_opts = [ - 'chdir=', - 'file=', - 'fmt=', - 'format=', - 'func=', - 'function=', - 'help', - 'prefix=', - 'tail=', - 'title=', - ] - - opts, args = getopt.getopt(argv[1:], short_opts, long_opts) - - for o, a in opts: - if o in ('-C', '--chdir'): - self.chdir = a - elif o in ('-f', '--file'): - self.config_file = a - elif o in ('--fmt', '--format'): - format = a - elif o in ('--func', '--function'): - function_name = a - elif o in ('-?', '-h', '--help'): - self.do_help(['help', 'func']) - sys.exit(0) - elif o in ('--max',): - max_time = int(a) - elif o in ('-p', '--prefix'): - self.prefix = a - elif o in ('-t', '--tail'): - tail = int(a) - elif o in ('--title',): - self.title = a - - if self.config_file: - with open(self.config_file, 'r') as f: - config = f.read() - exec(config, self.__dict__) - - if self.chdir: - os.chdir(self.chdir) - - if not args: - - pattern = '%s*.prof' % self.prefix - args = self.args_to_files([pattern], tail) - - if not args: - if self.chdir: - directory = self.chdir - else: - directory = os.getcwd() - - sys.stderr.write('%s: func: No arguments specified.\n' % self.name) - sys.stderr.write('%s No %s*.prof files found in "%s".\n' % (self.name_spaces, self.prefix, directory)) - sys.stderr.write('%s Type "%s help func" for help.\n' % (self.name_spaces, self.name)) - sys.exit(1) - - else: - - args = self.args_to_files(args, tail) - - cwd_ = os.getcwd() + os.sep - - if format == 'ascii': - - for file in args: - try: - f, line, func, time = \ - self.get_function_profile(file, function_name) - except ValueError as e: - sys.stderr.write("%s: func: %s: %s\n" % - (self.name, file, e)) - else: - if f.startswith(cwd_): - f = f[len(cwd_):] - print("%.3f %s:%d(%s)" % (time, f, line, func)) - - elif format == 'gnuplot': - - results = self.collect_results(args, self.get_function_time, - function_name) - - self.gnuplot_results(results) - - else: - - sys.stderr.write('%s: func: Unknown format "%s".\n' % (self.name, format)) - sys.exit(1) - - # - - def help_mem(self): - help = """\ - Usage: scons-time mem [OPTIONS] FILE [...] - - -C DIR, --chdir=DIR Change to DIR before looking for files - -f FILE, --file=FILE Read configuration from specified FILE - --fmt=FORMAT, --format=FORMAT Print data in specified FORMAT - -h, --help Print this help and exit - -p STRING, --prefix=STRING Use STRING as log file/profile prefix - --stage=STAGE Plot memory at the specified stage: - pre-read, post-read, pre-build, - post-build (default: post-build) - -t NUMBER, --tail=NUMBER Only report the last NUMBER files - --title=TITLE Specify the output plot TITLE - """ - sys.stdout.write(self.outdent(help)) - sys.stdout.flush() - - def do_mem(self, argv): - - format = 'ascii' - logfile_path = lambda x: x - stage = self.default_stage - tail = None - - short_opts = '?C:f:hp:t:' - - long_opts = [ - 'chdir=', - 'file=', - 'fmt=', - 'format=', - 'help', - 'prefix=', - 'stage=', - 'tail=', - 'title=', - ] - - opts, args = getopt.getopt(argv[1:], short_opts, long_opts) - - for o, a in opts: - if o in ('-C', '--chdir'): - self.chdir = a - elif o in ('-f', '--file'): - self.config_file = a - elif o in ('--fmt', '--format'): - format = a - elif o in ('-?', '-h', '--help'): - self.do_help(['help', 'mem']) - sys.exit(0) - elif o in ('-p', '--prefix'): - self.prefix = a - elif o in ('--stage',): - if a not in self.stages: - sys.stderr.write('%s: mem: Unrecognized stage "%s".\n' % (self.name, a)) - sys.exit(1) - stage = a - elif o in ('-t', '--tail'): - tail = int(a) - elif o in ('--title',): - self.title = a - - if self.config_file: - with open(self.config_file, 'r') as f: - config = f.read() - HACK_for_exec(config, self.__dict__) - - if self.chdir: - os.chdir(self.chdir) - logfile_path = lambda x: os.path.join(self.chdir, x) - - if not args: - - pattern = '%s*.log' % self.prefix - args = self.args_to_files([pattern], tail) - - if not args: - if self.chdir: - directory = self.chdir - else: - directory = os.getcwd() - - sys.stderr.write('%s: mem: No arguments specified.\n' % self.name) - sys.stderr.write('%s No %s*.log files found in "%s".\n' % (self.name_spaces, self.prefix, directory)) - sys.stderr.write('%s Type "%s help mem" for help.\n' % (self.name_spaces, self.name)) - sys.exit(1) - - else: - - args = self.args_to_files(args, tail) - - cwd_ = os.getcwd() + os.sep - - if format == 'ascii': - - self.ascii_table(args, tuple(self.stages), self.get_memory, logfile_path) - - elif format == 'gnuplot': - - results = self.collect_results(args, self.get_memory, - self.stage_strings[stage]) - - self.gnuplot_results(results) - - else: - - sys.stderr.write('%s: mem: Unknown format "%s".\n' % (self.name, format)) - sys.exit(1) - - return 0 - - # - - def help_obj(self): - help = """\ - Usage: scons-time obj [OPTIONS] OBJECT FILE [...] - - -C DIR, --chdir=DIR Change to DIR before looking for files - -f FILE, --file=FILE Read configuration from specified FILE - --fmt=FORMAT, --format=FORMAT Print data in specified FORMAT - -h, --help Print this help and exit - -p STRING, --prefix=STRING Use STRING as log file/profile prefix - --stage=STAGE Plot memory at the specified stage: - pre-read, post-read, pre-build, - post-build (default: post-build) - -t NUMBER, --tail=NUMBER Only report the last NUMBER files - --title=TITLE Specify the output plot TITLE - """ - sys.stdout.write(self.outdent(help)) - sys.stdout.flush() - - def do_obj(self, argv): - - format = 'ascii' - logfile_path = lambda x: x - stage = self.default_stage - tail = None - - short_opts = '?C:f:hp:t:' - - long_opts = [ - 'chdir=', - 'file=', - 'fmt=', - 'format=', - 'help', - 'prefix=', - 'stage=', - 'tail=', - 'title=', - ] - - opts, args = getopt.getopt(argv[1:], short_opts, long_opts) - - for o, a in opts: - if o in ('-C', '--chdir'): - self.chdir = a - elif o in ('-f', '--file'): - self.config_file = a - elif o in ('--fmt', '--format'): - format = a - elif o in ('-?', '-h', '--help'): - self.do_help(['help', 'obj']) - sys.exit(0) - elif o in ('-p', '--prefix'): - self.prefix = a - elif o in ('--stage',): - if a not in self.stages: - sys.stderr.write('%s: obj: Unrecognized stage "%s".\n' % (self.name, a)) - sys.stderr.write('%s Type "%s help obj" for help.\n' % (self.name_spaces, self.name)) - sys.exit(1) - stage = a - elif o in ('-t', '--tail'): - tail = int(a) - elif o in ('--title',): - self.title = a - - if not args: - sys.stderr.write('%s: obj: Must specify an object name.\n' % self.name) - sys.stderr.write('%s Type "%s help obj" for help.\n' % (self.name_spaces, self.name)) - sys.exit(1) - - object_name = args.pop(0) - - if self.config_file: - with open(self.config_file, 'r') as f: - config = f.read() - HACK_for_exec(config, self.__dict__) - - if self.chdir: - os.chdir(self.chdir) - logfile_path = lambda x: os.path.join(self.chdir, x) - - if not args: - - pattern = '%s*.log' % self.prefix - args = self.args_to_files([pattern], tail) - - if not args: - if self.chdir: - directory = self.chdir - else: - directory = os.getcwd() - - sys.stderr.write('%s: obj: No arguments specified.\n' % self.name) - sys.stderr.write('%s No %s*.log files found in "%s".\n' % (self.name_spaces, self.prefix, directory)) - sys.stderr.write('%s Type "%s help obj" for help.\n' % (self.name_spaces, self.name)) - sys.exit(1) - - else: - - args = self.args_to_files(args, tail) - - cwd_ = os.getcwd() + os.sep - - if format == 'ascii': - - self.ascii_table(args, tuple(self.stages), self.get_object_counts, logfile_path, object_name) - - elif format == 'gnuplot': - - stage_index = 0 - for s in self.stages: - if stage == s: - break - stage_index = stage_index + 1 - - results = self.collect_results(args, self.get_object_counts, - object_name, stage_index) - - self.gnuplot_results(results) - - else: - - sys.stderr.write('%s: obj: Unknown format "%s".\n' % (self.name, format)) - sys.exit(1) - - return 0 - - # - - def help_run(self): - help = """\ - Usage: scons-time run [OPTIONS] [FILE ...] - - --chdir=DIR Name of unpacked directory for chdir - -f FILE, --file=FILE Read configuration from specified FILE - -h, --help Print this help and exit - -n, --no-exec No execute, just print command lines - --number=NUMBER Put output in files for run NUMBER - --outdir=OUTDIR Put output files in OUTDIR - -p STRING, --prefix=STRING Use STRING as log file/profile prefix - --python=PYTHON Time using the specified PYTHON - -q, --quiet Don't print command lines - --scons=SCONS Time using the specified SCONS - --svn=URL, --subversion=URL Use SCons from Subversion URL - -v, --verbose Display output of commands - """ - sys.stdout.write(self.outdent(help)) - sys.stdout.flush() - - def do_run(self, argv): - """ - """ - run_number_list = [None] - - short_opts = '?f:hnp:qs:v' - - long_opts = [ - 'file=', - 'help', - 'no-exec', - 'number=', - 'outdir=', - 'prefix=', - 'python=', - 'quiet', - 'scons=', - 'svn=', - 'subdir=', - 'subversion=', - 'verbose', - ] - - opts, args = getopt.getopt(argv[1:], short_opts, long_opts) - - for o, a in opts: - if o in ('-f', '--file'): - self.config_file = a - elif o in ('-?', '-h', '--help'): - self.do_help(['help', 'run']) - sys.exit(0) - elif o in ('-n', '--no-exec'): - self.execute = self._do_not_execute - elif o in ('--number',): - run_number_list = self.split_run_numbers(a) - elif o in ('--outdir',): - self.outdir = a - elif o in ('-p', '--prefix'): - self.prefix = a - elif o in ('--python',): - self.python = a - elif o in ('-q', '--quiet'): - self.display = self._do_not_display - elif o in ('-s', '--subdir'): - self.subdir = a - elif o in ('--scons',): - self.scons = a - elif o in ('--svn', '--subversion'): - self.subversion_url = a - elif o in ('-v', '--verbose'): - self.redirect = tee_to_file - self.verbose = True - self.svn_co_flag = '' - - if not args and not self.config_file: - sys.stderr.write('%s: run: No arguments or -f config file specified.\n' % self.name) - sys.stderr.write('%s Type "%s help run" for help.\n' % (self.name_spaces, self.name)) - sys.exit(1) - - if self.config_file: - with open(self.config_file, 'r') as f: - config = f.read() - exec(config, self.__dict__) - - if args: - self.archive_list = args - - archive_file_name = os.path.split(self.archive_list[0])[1] - - if not self.subdir: - self.subdir = self.archive_splitext(archive_file_name)[0] - - if not self.prefix: - self.prefix = self.archive_splitext(archive_file_name)[0] - - prepare = None - if self.subversion_url: - prepare = self.prep_subversion_run - - for run_number in run_number_list: - self.individual_run(run_number, self.archive_list, prepare) - - def split_run_numbers(self, s): - result = [] - for n in s.split(','): - try: - x, y = n.split('-') - except ValueError: - result.append(int(n)) - else: - result.extend(list(range(int(x), int(y)+1))) - return result - - def scons_path(self, dir): - return os.path.join(dir, 'src', 'script', 'scons.py') - - def scons_lib_dir_path(self, dir): - return os.path.join(dir, 'src', 'engine') - - def prep_subversion_run(self, commands, removals): - self.svn_tmpdir = tempfile.mkdtemp(prefix=self.name + '-svn-') - removals.append((shutil.rmtree, 'rm -rf %%s', self.svn_tmpdir)) - - self.scons = self.scons_path(self.svn_tmpdir) - self.scons_lib_dir = self.scons_lib_dir_path(self.svn_tmpdir) - - commands.extend([ - '%(svn)s co %(svn_co_flag)s -r %(run_number)s %(subversion_url)s %(svn_tmpdir)s', - ]) - - def individual_run(self, run_number, archive_list, prepare=None): - """ - Performs an individual run of the default SCons invocations. - """ - - commands = [] - removals = [] - - if prepare: - prepare(commands, removals) - - save_scons = self.scons - save_scons_wrapper = self.scons_wrapper - save_scons_lib_dir = self.scons_lib_dir - - if self.outdir is None: - self.outdir = self.orig_cwd - elif not os.path.isabs(self.outdir): - self.outdir = os.path.join(self.orig_cwd, self.outdir) - - if self.scons is None: - self.scons = self.scons_path(self.orig_cwd) - - if self.scons_lib_dir is None: - self.scons_lib_dir = self.scons_lib_dir_path(self.orig_cwd) - - if self.scons_wrapper is None: - self.scons_wrapper = self.scons - - if not run_number: - run_number = self.find_next_run_number(self.outdir, self.prefix) - - self.run_number = str(run_number) - - self.prefix_run = self.prefix + '-%03d' % run_number - - if self.targets0 is None: - self.targets0 = self.startup_targets - if self.targets1 is None: - self.targets1 = self.targets - if self.targets2 is None: - self.targets2 = self.targets - - self.tmpdir = tempfile.mkdtemp(prefix=self.name + '-') - - commands.extend([ - (os.chdir, 'cd %%s', self.tmpdir), - ]) - - for archive in archive_list: - if not os.path.isabs(archive): - archive = os.path.join(self.orig_cwd, archive) - if os.path.isdir(archive): - dest = os.path.split(archive)[1] - commands.append((shutil.copytree, 'cp -r %%s %%s', archive, dest)) - else: - suffix = self.archive_splitext(archive)[1] - unpack_command = self.unpack_map.get(suffix) - if not unpack_command: - dest = os.path.split(archive)[1] - commands.append((shutil.copyfile, 'cp %%s %%s', archive, dest)) - else: - commands.append(unpack_command + (archive,)) - - commands.extend([ - (os.chdir, 'cd %%s', self.subdir), - ]) - - commands.extend(self.initial_commands) - - commands.extend([ - (lambda: read_tree('.'), - 'find * -type f | xargs cat > /dev/null'), - - (self.set_env, 'export %%s=%%s', - 'SCONS_LIB_DIR', self.scons_lib_dir), - - '%(python)s %(scons_wrapper)s --version', - ]) - - index = 0 - for run_command in self.run_commands: - setattr(self, 'prof%d' % index, self.profile_name(index)) - c = ( - self.log_execute, - self.log_display, - run_command, - self.logfile_name(index), - ) - commands.append(c) - index = index + 1 - - commands.extend([ - (os.chdir, 'cd %%s', self.orig_cwd), - ]) - - if not os.environ.get('PRESERVE'): - commands.extend(removals) - commands.append((shutil.rmtree, 'rm -rf %%s', self.tmpdir)) - - self.run_command_list(commands, self.__dict__) - - self.scons = save_scons - self.scons_lib_dir = save_scons_lib_dir - self.scons_wrapper = save_scons_wrapper - - # - - def help_time(self): - help = """\ - Usage: scons-time time [OPTIONS] FILE [...] - - -C DIR, --chdir=DIR Change to DIR before looking for files - -f FILE, --file=FILE Read configuration from specified FILE - --fmt=FORMAT, --format=FORMAT Print data in specified FORMAT - -h, --help Print this help and exit - -p STRING, --prefix=STRING Use STRING as log file/profile prefix - -t NUMBER, --tail=NUMBER Only report the last NUMBER files - --which=TIMER Plot timings for TIMER: total, - SConscripts, SCons, commands. - """ - sys.stdout.write(self.outdent(help)) - sys.stdout.flush() - - def do_time(self, argv): - - format = 'ascii' - logfile_path = lambda x: x - tail = None - which = 'total' - - short_opts = '?C:f:hp:t:' - - long_opts = [ - 'chdir=', - 'file=', - 'fmt=', - 'format=', - 'help', - 'prefix=', - 'tail=', - 'title=', - 'which=', - ] - - opts, args = getopt.getopt(argv[1:], short_opts, long_opts) - - for o, a in opts: - if o in ('-C', '--chdir'): - self.chdir = a - elif o in ('-f', '--file'): - self.config_file = a - elif o in ('--fmt', '--format'): - format = a - elif o in ('-?', '-h', '--help'): - self.do_help(['help', 'time']) - sys.exit(0) - elif o in ('-p', '--prefix'): - self.prefix = a - elif o in ('-t', '--tail'): - tail = int(a) - elif o in ('--title',): - self.title = a - elif o in ('--which',): - if a not in list(self.time_strings.keys()): - sys.stderr.write('%s: time: Unrecognized timer "%s".\n' % (self.name, a)) - sys.stderr.write('%s Type "%s help time" for help.\n' % (self.name_spaces, self.name)) - sys.exit(1) - which = a - - if self.config_file: - with open(self.config_file, 'r') as f: - config = f.read() - HACK_for_exec(config, self.__dict__) - - if self.chdir: - os.chdir(self.chdir) - logfile_path = lambda x: os.path.join(self.chdir, x) - - if not args: - - pattern = '%s*.log' % self.prefix - args = self.args_to_files([pattern], tail) - - if not args: - if self.chdir: - directory = self.chdir - else: - directory = os.getcwd() - - sys.stderr.write('%s: time: No arguments specified.\n' % self.name) - sys.stderr.write('%s No %s*.log files found in "%s".\n' % (self.name_spaces, self.prefix, directory)) - sys.stderr.write('%s Type "%s help time" for help.\n' % (self.name_spaces, self.name)) - sys.exit(1) - - else: - - args = self.args_to_files(args, tail) - - cwd_ = os.getcwd() + os.sep - - if format == 'ascii': - - columns = ("Total", "SConscripts", "SCons", "commands") - self.ascii_table(args, columns, self.get_debug_times, logfile_path) - - elif format == 'gnuplot': - - results = self.collect_results(args, self.get_debug_times, - self.time_strings[which]) - - self.gnuplot_results(results, fmt='%s %.6f') - - else: - - sys.stderr.write('%s: time: Unknown format "%s".\n' % (self.name, format)) - sys.exit(1) - -if __name__ == '__main__': - opts, args = getopt.getopt(sys.argv[1:], 'h?V', ['help', 'version']) - - ST = SConsTimer() - - for o, a in opts: - if o in ('-?', '-h', '--help'): - ST.do_help(['help']) - sys.exit(0) - elif o in ('-V', '--version'): - sys.stdout.write('scons-time version\n') - sys.exit(0) - - if not args: - sys.stderr.write('Type "%s help" for usage.\n' % ST.name) - sys.exit(1) - - ST.execute_subcommand(args) - -# Local Variables: -# tab-width:4 -# indent-tabs-mode:nil -# End: -# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/src/script/scons.bat b/src/script/scons.bat deleted file mode 100644 index 10b8637..0000000 --- a/src/script/scons.bat +++ /dev/null @@ -1,38 +0,0 @@ -@REM __COPYRIGHT__ -@REM __FILE__ __REVISION__ __DATE__ __DEVELOPER__ -@echo off -set SCONS_ERRORLEVEL= -if "%OS%" == "Windows_NT" goto WinNT - -@REM for 9x/Me you better not have more than 9 args -python -c "from os.path import join; import sys; sys.path = [ join(sys.prefix, 'Lib', 'site-packages', 'scons-__VERSION__'), join(sys.prefix, 'Lib', 'site-packages', 'scons'), join(sys.prefix, 'scons-__VERSION__'), join(sys.prefix, 'scons')] + sys.path; import SCons.Script; SCons.Script.main()" %1 %2 %3 %4 %5 %6 %7 %8 %9 -@REM no way to set exit status of this script for 9x/Me -goto endscons - -@REM Credit where credit is due: we return the exit code despite our -@REM use of setlocal+endlocal using a technique from Bear's Journal: -@REM http://code-bear.com/bearlog/2007/06/01/getting-the-exit-code-from-a-batch-file-that-is-run-from-a-python-program/ - -:WinNT -setlocal -@REM ensure the script will be executed with the Python it was installed for -pushd %~dp0.. -set path=%~dp0;%CD%;%path% -popd -@REM try the script named as the .bat file in current dir, then in Scripts subdir -set scriptname=%~dp0%~n0.py -if not exist "%scriptname%" set scriptname=%~dp0Scripts\%~n0.py -@REM Handle when running from wheel where the script has no .py extension -if not exist "%scriptname%" set scriptname=%~dp0%~n0 -python "%scriptname%" %* -endlocal & set SCONS_ERRORLEVEL=%ERRORLEVEL% - -if NOT "%COMSPEC%" == "%SystemRoot%\system32\cmd.exe" goto returncode -if errorlevel 9009 echo you do not have python in your PATH -goto endscons - -:returncode -exit /B %SCONS_ERRORLEVEL% - -:endscons -call :returncode %SCONS_ERRORLEVEL% diff --git a/src/script/scons.py b/src/script/scons.py deleted file mode 100755 index 1e12898..0000000 --- a/src/script/scons.py +++ /dev/null @@ -1,208 +0,0 @@ -#! /usr/bin/env python -# -# SCons - a Software Constructor -# -# __COPYRIGHT__ -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - -__version__ = "__VERSION__" - -__build__ = "__BUILD__" - -__buildsys__ = "__BUILDSYS__" - -__date__ = "__DATE__" - -__developer__ = "__DEVELOPER__" - -# This is the entry point to the SCons program. -# The only job of this script is to work out where the guts of the program -# could be and import them, where the real work begins. -# SCons can be invoked several different ways -# - from an installed location -# - from a "local install" copy -# - from a source tree, which has a different dir struture than the other two -# Try to account for all those possibilities. - -import os -import sys - -############################################################################## -# BEGIN STANDARD SCons SCRIPT HEADER -# -# This is the cut-and-paste logic so that a self-contained script can -# interoperate correctly with different SCons versions and installation -# locations for the engine. If you modify anything in this section, you -# should also change other scripts that use this same header. -############################################################################## - -# compatibility check -if sys.version_info < (3,5,0): - msg = "scons: *** SCons version %s does not run under Python version %s.\n\ -Python >= 3.5 is required.\n" - sys.stderr.write(msg % (__version__, sys.version.split()[0])) - sys.exit(1) - -# Strip the script directory from sys.path so on case-insensitive -# (WIN32) systems Python doesn't think that the "scons" script is the -# "SCons" package. -script_dir = os.path.dirname(os.path.realpath(__file__)) -script_path = os.path.realpath(os.path.dirname(__file__)) -if script_path in sys.path: - sys.path.remove(script_path) - -libs = [] - -if "SCONS_LIB_DIR" in os.environ: - libs.append(os.environ["SCONS_LIB_DIR"]) - -# running from source takes 2nd priority (since 2.3.2), following SCONS_LIB_DIR -source_path = os.path.join(script_path, os.pardir, 'engine') -if os.path.isdir(source_path): - libs.append(source_path) - -# add local-install locations -local_version = 'scons-local-' + __version__ -local = 'scons-local' -if script_dir: - local_version = os.path.join(script_dir, local_version) - local = os.path.join(script_dir, local) -if os.path.isdir(local_version): - libs.append(os.path.abspath(local_version)) -if os.path.isdir(local): - libs.append(os.path.abspath(local)) - -scons_version = 'scons-%s' % __version__ - -# preferred order of scons lookup paths -prefs = [] - -# if we can find package information, use it -try: - import pkg_resources -except ImportError: - pass -else: - try: - d = pkg_resources.get_distribution('scons') - except pkg_resources.DistributionNotFound: - pass - else: - prefs.append(d.location) - -if sys.platform == 'win32': - # Use only sys.prefix on Windows - prefs.append(sys.prefix) - prefs.append(os.path.join(sys.prefix, 'Lib', 'site-packages')) -else: - # On other (POSIX) platforms, things are more complicated due to - # the variety of path names and library locations. - # Build up some possibilities, then transform them into candidates - temp = [] - if script_dir == 'bin': - # script_dir is `pwd`/bin; - # check `pwd`/lib/scons*. - temp.append(os.getcwd()) - else: - if script_dir == '.' or script_dir == '': - script_dir = os.getcwd() - head, tail = os.path.split(script_dir) - if tail == "bin": - # script_dir is /foo/bin; - # check /foo/lib/scons*. - temp.append(head) - - head, tail = os.path.split(sys.prefix) - if tail == "usr": - # sys.prefix is /foo/usr; - # check /foo/usr/lib/scons* first, - # then /foo/usr/local/lib/scons*. - temp.append(sys.prefix) - temp.append(os.path.join(sys.prefix, "local")) - elif tail == "local": - h, t = os.path.split(head) - if t == "usr": - # sys.prefix is /foo/usr/local; - # check /foo/usr/local/lib/scons* first, - # then /foo/usr/lib/scons*. - temp.append(sys.prefix) - temp.append(head) - else: - # sys.prefix is /foo/local; - # check only /foo/local/lib/scons*. - temp.append(sys.prefix) - else: - # sys.prefix is /foo (ends in neither /usr or /local); - # check only /foo/lib/scons*. - temp.append(sys.prefix) - - # suffix these to add to our original prefs: - prefs.extend([os.path.join(x, 'lib') for x in temp]) - prefs.extend([os.path.join(x, 'lib', 'python' + sys.version[:3], - 'site-packages') for x in temp]) - - - # Add the parent directory of the current python's library to the - # preferences. This picks up differences between, e.g., lib and lib64, - # and finds the base location in case of a non-copying virtualenv. - try: - libpath = os.__file__ - except AttributeError: - pass - else: - # Split /usr/libfoo/python*/os.py to /usr/libfoo/python*. - libpath, tail = os.path.split(libpath) - # Split /usr/libfoo/python* to /usr/libfoo - libpath, tail = os.path.split(libpath) - # Check /usr/libfoo/scons*. - prefs.append(libpath) - -# Look first for 'scons-__version__' in all of our preference libs, -# then for 'scons'. Skip paths that do not exist. -libs.extend([os.path.join(x, scons_version) for x in prefs if os.path.isdir(x)]) -libs.extend([os.path.join(x, 'scons') for x in prefs if os.path.isdir(x)]) - -sys.path = libs + sys.path - -############################################################################## -# END STANDARD SCons SCRIPT HEADER -############################################################################## - -if __name__ == "__main__": - try: - import SCons.Script - except ImportError: - sys.stderr.write("SCons import failed. Unable to find engine files in:\n") - for path in libs: - sys.stderr.write(" {}\n".format(path)) - raise - - # this does all the work, and calls sys.exit - # with the proper exit status when done. - SCons.Script.main() - -# Local Variables: -# tab-width:4 -# indent-tabs-mode:nil -# End: -# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/src/script/sconsign.py b/src/script/sconsign.py deleted file mode 100644 index 726838c..0000000 --- a/src/script/sconsign.py +++ /dev/null @@ -1,652 +0,0 @@ -#! /usr/bin/env python -# -# SCons - a Software Constructor -# -# __COPYRIGHT__ -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - -__version__ = "__VERSION__" - -__build__ = "__BUILD__" - -__buildsys__ = "__BUILDSYS__" - -__date__ = "__DATE__" - -__developer__ = "__DEVELOPER__" - -import os -import sys - -############################################################################## -# BEGIN STANDARD SCons SCRIPT HEADER -# -# This is the cut-and-paste logic so that a self-contained script can -# interoperate correctly with different SCons versions and installation -# locations for the engine. If you modify anything in this section, you -# should also change other scripts that use this same header. -############################################################################## - -# compatibility check -if sys.version_info < (3,5,0): - msg = "scons: *** SCons version %s does not run under Python version %s.\n\ -Python >= 3.5 is required.\n" - sys.stderr.write(msg % (__version__, sys.version.split()[0])) - sys.exit(1) - -# Strip the script directory from sys.path so on case-insensitive -# (WIN32) systems Python doesn't think that the "scons" script is the -# "SCons" package. -script_dir = os.path.dirname(os.path.realpath(__file__)) -script_path = os.path.realpath(os.path.dirname(__file__)) -if script_path in sys.path: - sys.path.remove(script_path) - -libs = [] - -if "SCONS_LIB_DIR" in os.environ: - libs.append(os.environ["SCONS_LIB_DIR"]) - -# running from source takes 2nd priority (since 2.3.2), following SCONS_LIB_DIR -source_path = os.path.join(script_path, os.pardir, 'engine') -if os.path.isdir(source_path): - libs.append(source_path) - -# add local-install locations -local_version = 'scons-local-' + __version__ -local = 'scons-local' -if script_dir: - local_version = os.path.join(script_dir, local_version) - local = os.path.join(script_dir, local) -if os.path.isdir(local_version): - libs.append(os.path.abspath(local_version)) -if os.path.isdir(local): - libs.append(os.path.abspath(local)) - -scons_version = 'scons-%s' % __version__ - -# preferred order of scons lookup paths -prefs = [] - -# if we can find package information, use it -try: - import pkg_resources -except ImportError: - pass -else: - try: - d = pkg_resources.get_distribution('scons') - except pkg_resources.DistributionNotFound: - pass - else: - prefs.append(d.location) - -if sys.platform == 'win32': - # Use only sys.prefix on Windows - prefs.append(sys.prefix) - prefs.append(os.path.join(sys.prefix, 'Lib', 'site-packages')) -else: - # On other (POSIX) platforms, things are more complicated due to - # the variety of path names and library locations. - # Build up some possibilities, then transform them into candidates - temp = [] - if script_dir == 'bin': - # script_dir is `pwd`/bin; - # check `pwd`/lib/scons*. - temp.append(os.getcwd()) - else: - if script_dir in ('.', ''): - script_dir = os.getcwd() - head, tail = os.path.split(script_dir) - if tail == "bin": - # script_dir is /foo/bin; - # check /foo/lib/scons*. - temp.append(head) - - head, tail = os.path.split(sys.prefix) - if tail == "usr": - # sys.prefix is /foo/usr; - # check /foo/usr/lib/scons* first, - # then /foo/usr/local/lib/scons*. - temp.append(sys.prefix) - temp.append(os.path.join(sys.prefix, "local")) - elif tail == "local": - h, t = os.path.split(head) - if t == "usr": - # sys.prefix is /foo/usr/local; - # check /foo/usr/local/lib/scons* first, - # then /foo/usr/lib/scons*. - temp.append(sys.prefix) - temp.append(head) - else: - # sys.prefix is /foo/local; - # check only /foo/local/lib/scons*. - temp.append(sys.prefix) - else: - # sys.prefix is /foo (ends in neither /usr or /local); - # check only /foo/lib/scons*. - temp.append(sys.prefix) - - # suffix these to add to our original prefs: - prefs.extend([os.path.join(x, 'lib') for x in temp]) - prefs.extend([os.path.join(x, 'lib', 'python' + sys.version[:3], - 'site-packages') for x in temp]) - - - # Add the parent directory of the current python's library to the - # preferences. This picks up differences between, e.g., lib and lib64, - # and finds the base location in case of a non-copying virtualenv. - try: - libpath = os.__file__ - except AttributeError: - pass - else: - # Split /usr/libfoo/python*/os.py to /usr/libfoo/python*. - libpath, _ = os.path.split(libpath) - # Split /usr/libfoo/python* to /usr/libfoo - libpath, tail = os.path.split(libpath) - # Check /usr/libfoo/scons*. - prefs.append(libpath) - -# Look first for 'scons-__version__' in all of our preference libs, -# then for 'scons'. Skip paths that do not exist. -libs.extend([os.path.join(x, scons_version) for x in prefs if os.path.isdir(x)]) -libs.extend([os.path.join(x, 'scons') for x in prefs if os.path.isdir(x)]) - -sys.path = libs + sys.path - -############################################################################## -# END STANDARD SCons SCRIPT HEADER -############################################################################## - -import SCons.compat - -try: - import whichdb - - whichdb = whichdb.whichdb -except ImportError as e: - from dbm import whichdb - -import time -import pickle - -import SCons.SConsign - - -def my_whichdb(filename): - if filename[-7:] == ".dblite": - return "SCons.dblite" - try: - with open(filename + ".dblite", "rb"): - return "SCons.dblite" - except IOError: - pass - return _orig_whichdb(filename) - - -# Should work on python2 -_orig_whichdb = whichdb -whichdb = my_whichdb - -# was changed for python3 -#_orig_whichdb = whichdb.whichdb -#dbm.whichdb = my_whichdb - -def my_import(mname): - import imp - - if '.' in mname: - i = mname.rfind('.') - parent = my_import(mname[:i]) - fp, pathname, description = imp.find_module(mname[i+1:], - parent.__path__) - else: - fp, pathname, description = imp.find_module(mname) - return imp.load_module(mname, fp, pathname, description) - - -class Flagger(object): - default_value = 1 - - def __setitem__(self, item, value): - self.__dict__[item] = value - self.default_value = 0 - - def __getitem__(self, item): - return self.__dict__.get(item, self.default_value) - - -Do_Call = None -Print_Directories = [] -Print_Entries = [] -Print_Flags = Flagger() -Verbose = 0 -Readable = 0 -Warns = 0 - - -def default_mapper(entry, name): - """ - Stringify an entry that doesn't have an explicit mapping. - - Args: - entry: entry - name: field name - - Returns: str - - """ - try: - val = eval("entry." + name) - except AttributeError: - val = None - if sys.version_info.major >= 3 and isinstance(val, bytes): - # This is a dirty hack for py 2/3 compatibility. csig is a bytes object - # in Python3 while Python2 bytes are str. Hence, we decode the csig to a - # Python3 string - val = val.decode() - return str(val) - - -def map_action(entry, _): - """ - Stringify an action entry and signature. - - Args: - entry: action entry - second argument is not used - - Returns: str - - """ - try: - bact = entry.bact - bactsig = entry.bactsig - except AttributeError: - return None - return '%s [%s]' % (bactsig, bact) - - -def map_timestamp(entry, _): - """ - Stringify a timestamp entry. - - Args: - entry: timestamp entry - second argument is not used - - Returns: str - - """ - try: - timestamp = entry.timestamp - except AttributeError: - timestamp = None - if Readable and timestamp: - return "'" + time.ctime(timestamp) + "'" - else: - return str(timestamp) - - -def map_bkids(entry, _): - """ - Stringify an implicit entry. - - Args: - entry: - second argument is not used - - Returns: str - - """ - try: - bkids = entry.bsources + entry.bdepends + entry.bimplicit - bkidsigs = entry.bsourcesigs + entry.bdependsigs + entry.bimplicitsigs - except AttributeError: - return None - - if len(bkids) != len(bkidsigs): - global Warns - Warns += 1 - # add warning to result rather than direct print so it will line up - msg = "Warning: missing information, {} ids but {} sigs" - result = [msg.format(len(bkids), len(bkidsigs))] - else: - result = [] - result += [nodeinfo_string(bkid, bkidsig, " ") - for bkid, bkidsig in zip(bkids, bkidsigs)] - if not result: - return None - return "\n ".join(result) - - -map_field = { - 'action' : map_action, - 'timestamp' : map_timestamp, - 'bkids' : map_bkids, -} - -map_name = { - 'implicit' : 'bkids', -} - - -def field(name, entry, verbose=Verbose): - if not Print_Flags[name]: - return None - fieldname = map_name.get(name, name) - mapper = map_field.get(fieldname, default_mapper) - val = mapper(entry, name) - if verbose: - val = name + ": " + val - return val - - -def nodeinfo_raw(name, ninfo, prefix=""): - # This just formats the dictionary, which we would normally use str() - # to do, except that we want the keys sorted for deterministic output. - d = ninfo.__getstate__() - try: - keys = ninfo.field_list + ['_version_id'] - except AttributeError: - keys = sorted(d.keys()) - l = [] - for k in keys: - l.append('%s: %s' % (repr(k), repr(d.get(k)))) - if '\n' in name: - name = repr(name) - return name + ': {' + ', '.join(l) + '}' - - -def nodeinfo_cooked(name, ninfo, prefix=""): - try: - field_list = ninfo.field_list - except AttributeError: - field_list = [] - if '\n' in name: - name = repr(name) - outlist = [name + ':'] + [ - f for f in [field(x, ninfo, Verbose) for x in field_list] if f - ] - if Verbose: - sep = '\n ' + prefix - else: - sep = ' ' - return sep.join(outlist) - - -nodeinfo_string = nodeinfo_cooked - - -def printfield(name, entry, prefix=""): - outlist = field("implicit", entry, 0) - if outlist: - if Verbose: - print(" implicit:") - print(" " + outlist) - outact = field("action", entry, 0) - if outact: - if Verbose: - print(" action: " + outact) - else: - print(" " + outact) - - -def printentries(entries, location): - if Print_Entries: - for name in Print_Entries: - try: - entry = entries[name] - except KeyError: - err = "sconsign: no entry `%s' in `%s'\n" % (name, location) - sys.stderr.write(err) - else: - try: - ninfo = entry.ninfo - except AttributeError: - print(name + ":") - else: - print(nodeinfo_string(name, entry.ninfo)) - printfield(name, entry.binfo) - else: - for name in sorted(entries.keys()): - entry = entries[name] - try: - ninfo = entry.ninfo - except AttributeError: - print(name + ":") - else: - print(nodeinfo_string(name, entry.ninfo)) - printfield(name, entry.binfo) - - -class Do_SConsignDB(object): - def __init__(self, dbm_name, dbm): - self.dbm_name = dbm_name - self.dbm = dbm - - def __call__(self, fname): - # The *dbm modules stick their own file suffixes on the names - # that are passed in. This causes us to jump through some - # hoops here. - try: - # Try opening the specified file name. Example: - # SPECIFIED OPENED BY self.dbm.open() - # --------- ------------------------- - # .sconsign => .sconsign.dblite - # .sconsign.dblite => .sconsign.dblite.dblite - db = self.dbm.open(fname, "r") - except (IOError, OSError) as e: - print_e = e - try: - # That didn't work, so try opening the base name, - # so that if they actually passed in 'sconsign.dblite' - # (for example), the dbm module will put the suffix back - # on for us and open it anyway. - db = self.dbm.open(os.path.splitext(fname)[0], "r") - except (IOError, OSError): - # That didn't work either. See if the file name - # they specified even exists (independent of the dbm - # suffix-mangling). - try: - with open(fname, "rb"): - pass # this is a touch only, we don't use it here. - except (IOError, OSError) as e: - # Nope, that file doesn't even exist, so report that - # fact back. - print_e = e - sys.stderr.write("sconsign: %s\n" % print_e) - return - except KeyboardInterrupt: - raise - except pickle.UnpicklingError: - sys.stderr.write("sconsign: ignoring invalid `%s' file `%s'\n" - % (self.dbm_name, fname)) - return - except Exception as e: - sys.stderr.write("sconsign: ignoring invalid `%s' file `%s': %s\n" - % (self.dbm_name, fname, e)) - exc_type, _, _ = sys.exc_info() - if exc_type.__name__ == "ValueError" and sys.version_info < (3,0,0): - sys.stderr.write("Python 2 only supports pickle protocols 0-2.\n") - return - - if Print_Directories: - for dir in Print_Directories: - try: - val = db[dir] - except KeyError: - err = "sconsign: no dir `%s' in `%s'\n" % (dir, args[0]) - sys.stderr.write(err) - else: - self.printentries(dir, val) - else: - for dir in sorted(db.keys()): - self.printentries(dir, db[dir]) - - @staticmethod - def printentries(dir, val): - try: - print('=== ' + dir + ':') - except TypeError: - print('=== ' + dir.decode() + ':') - printentries(pickle.loads(val), dir) - - -def Do_SConsignDir(name): - try: - with open(name, 'rb') as fp: - try: - sconsign = SCons.SConsign.Dir(fp) - except KeyboardInterrupt: - raise - except pickle.UnpicklingError: - err = "sconsign: ignoring invalid .sconsign file `%s'\n" % name - sys.stderr.write(err) - return - except Exception as e: - err = "sconsign: ignoring invalid .sconsign file `%s': %s\n" % (name, e) - sys.stderr.write(err) - return - printentries(sconsign.entries, args[0]) - except (IOError, OSError) as e: - sys.stderr.write("sconsign: %s\n" % e) - return - - -############################################################################## - -import getopt - -helpstr = """\ -Usage: sconsign [OPTIONS] [FILE ...] -Options: - -a, --act, --action Print build action information. - -c, --csig Print content signature information. - -d DIR, --dir=DIR Print only info about DIR. - -e ENTRY, --entry=ENTRY Print only info about ENTRY. - -f FORMAT, --format=FORMAT FILE is in the specified FORMAT. - -h, --help Print this message and exit. - -i, --implicit Print implicit dependency information. - -r, --readable Print timestamps in human-readable form. - --raw Print raw Python object representations. - -s, --size Print file sizes. - -t, --timestamp Print timestamp information. - -v, --verbose Verbose, describe each field. -""" - -try: - opts, args = getopt.getopt(sys.argv[1:], "acd:e:f:hirstv", - ['act', 'action', - 'csig', 'dir=', 'entry=', - 'format=', 'help', 'implicit', - 'raw', 'readable', - 'size', 'timestamp', 'verbose']) -except getopt.GetoptError as err: - sys.stderr.write(str(err) + '\n') - print(helpstr) - sys.exit(2) - -for o, a in opts: - if o in ('-a', '--act', '--action'): - Print_Flags['action'] = 1 - elif o in ('-c', '--csig'): - Print_Flags['csig'] = 1 - elif o in ('-d', '--dir'): - Print_Directories.append(a) - elif o in ('-e', '--entry'): - Print_Entries.append(a) - elif o in ('-f', '--format'): - # Try to map the given DB format to a known module - # name, that we can then try to import... - Module_Map = {'dblite': 'SCons.dblite', 'sconsign': None} - dbm_name = Module_Map.get(a, a) - if dbm_name: - try: - if dbm_name != "SCons.dblite": - dbm = my_import(dbm_name) - else: - import SCons.dblite - - dbm = SCons.dblite - # Ensure that we don't ignore corrupt DB files, - # this was handled by calling my_import('SCons.dblite') - # again in earlier versions... - SCons.dblite.ignore_corrupt_dbfiles = 0 - except ImportError: - sys.stderr.write("sconsign: illegal file format `%s'\n" % a) - print(helpstr) - sys.exit(2) - Do_Call = Do_SConsignDB(a, dbm) - else: - Do_Call = Do_SConsignDir - elif o in ('-h', '--help'): - print(helpstr) - sys.exit(0) - elif o in ('-i', '--implicit'): - Print_Flags['implicit'] = 1 - elif o in ('--raw',): - nodeinfo_string = nodeinfo_raw - elif o in ('-r', '--readable'): - Readable = 1 - elif o in ('-s', '--size'): - Print_Flags['size'] = 1 - elif o in ('-t', '--timestamp'): - Print_Flags['timestamp'] = 1 - elif o in ('-v', '--verbose'): - Verbose = 1 - -if Do_Call: - for a in args: - Do_Call(a) -else: - if not args: - args = [".sconsign.dblite"] - for a in args: - dbm_name = whichdb(a) - if dbm_name: - Map_Module = {'SCons.dblite': 'dblite'} - if dbm_name != "SCons.dblite": - dbm = my_import(dbm_name) - else: - import SCons.dblite - - dbm = SCons.dblite - # Ensure that we don't ignore corrupt DB files, - # this was handled by calling my_import('SCons.dblite') - # again in earlier versions... - SCons.dblite.ignore_corrupt_dbfiles = 0 - Do_SConsignDB(Map_Module.get(dbm_name, dbm_name), dbm)(a) - else: - Do_SConsignDir(a) - - if Warns: - print("NOTE: there were %d warnings, please check output" % Warns) -sys.exit(0) - -# Local Variables: -# tab-width:4 -# indent-tabs-mode:nil -# End: -# vim: set expandtab tabstop=4 shiftwidth=4: -- cgit v0.12 From bdc3e10c5963399badd2833a2ae74a1f87e70b36 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 21 Mar 2020 16:36:52 -0700 Subject: update runtest.py to find scons in scripts --- runtest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtest.py b/runtest.py index 3edf36c..2ee4476 100755 --- a/runtest.py +++ b/runtest.py @@ -555,7 +555,7 @@ else: scons_runtest_dir = base if not external: - scons_script_dir = sd or os.path.join(base, 'src', 'script') + scons_script_dir = sd or os.path.join(base, 'script') scons_lib_dir = ld or os.path.join(base, 'src', 'engine') else: scons_script_dir = sd or '' -- cgit v0.12 From 2888b4f2e8ce13dab28277552ec9fe4ea222cc61 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 21 Mar 2020 17:03:08 -0700 Subject: update scripts to work from /scripts --- scripts/scons-configure-cache.py | 155 +++++++++--------- scripts/sconsign.py | 328 +++++++++++++-------------------------- 2 files changed, 192 insertions(+), 291 deletions(-) diff --git a/scripts/scons-configure-cache.py b/scripts/scons-configure-cache.py index 716315c..80783a9 100644 --- a/scripts/scons-configure-cache.py +++ b/scripts/scons-configure-cache.py @@ -100,78 +100,83 @@ config_entries = { } } -parser = argparse.ArgumentParser( - description='Modify the configuration of an scons cache directory', - epilog=''' - Unspecified options will not be changed unless they are not - set at all, in which case they are set to an appropriate default. - ''') - -parser.add_argument('cache-dir', help='Path to scons cache directory') -for param in config_entries: - parser.add_argument('--' + param.replace('_', '-'), - **config_entries[param]['command-line']) -parser.add_argument('--version', - action='version', - version='%(prog)s 1.0') -parser.add_argument('--show', - action="store_true", - help="show current configuration") - -# Get the command line as a dict without any of the unspecified entries. -args = dict([x for x in vars(parser.parse_args()).items() if x[1]]) - -# It seems somewhat strange to me, but positional arguments don't get the - -# in the name changed to _, whereas optional arguments do... -cache = args['cache-dir'] -if not os.path.isdir(cache): - raise RuntimeError("There is no cache directory named %s" % cache) -os.chdir(cache) -del args['cache-dir'] - -if not os.path.exists('config'): - # old config dirs did not have a 'config' file. Try to update. - # Validate the only files in the directory are directories 0-9, a-f - expected = ['{:X}'.format(x) for x in range(0, 16)] - if not set(os.listdir('.')).issubset(expected): - raise RuntimeError( - "%s does not look like a valid version 1 cache directory" % cache) - config = dict() -else: - with open('config') as conf: - config = json.load(conf) - -if args.get('show', None): - print("Current configuration in '%s':" % cache) - print(json.dumps(config, sort_keys=True, - indent=4, separators=(',', ': '))) - # in case of the show argument, emit some stats as well - file_count = 0 - for _, _, files in os.walk('.'): - file_count += len(files) - if file_count: # skip config file if it exists - file_count -= 1 - print("Cache contains %s files" % file_count) - del args['show'] - -# Find any keys that are not currently set but should be -for key in config_entries: - if key not in config: - if 'implicit' in config_entries[key]: - config[key] = config_entries[key]['implicit'] - else: - config[key] = config_entries[key]['default'] - if key not in args: - args[key] = config_entries[key]['default'] - -# Now go through each entry in args to see if it changes an existing config -# setting. -for key in args: - if args[key] != config[key]: - if 'converter' in config_entries[key]: - config_entries[key]['converter'](config[key], args[key]) - config[key] = args[key] - -# and write the updated config file -with open('config', 'w') as conf: - json.dump(config, conf) + +def main(): + parser = argparse.ArgumentParser( + description='Modify the configuration of an scons cache directory', + epilog=''' + Unspecified options will not be changed unless they are not + set at all, in which case they are set to an appropriate default. + ''') + + parser.add_argument('cache-dir', help='Path to scons cache directory') + for param in config_entries: + parser.add_argument('--' + param.replace('_', '-'), + **config_entries[param]['command-line']) + parser.add_argument('--version', + action='version', + version='%(prog)s 1.0') + parser.add_argument('--show', + action="store_true", + help="show current configuration") + + # Get the command line as a dict without any of the unspecified entries. + args = dict([x for x in vars(parser.parse_args()).items() if x[1]]) + + # It seems somewhat strange to me, but positional arguments don't get the - + # in the name changed to _, whereas optional arguments do... + cache = args['cache-dir'] + if not os.path.isdir(cache): + raise RuntimeError("There is no cache directory named %s" % cache) + os.chdir(cache) + del args['cache-dir'] + + if not os.path.exists('config'): + # old config dirs did not have a 'config' file. Try to update. + # Validate the only files in the directory are directories 0-9, a-f + expected = ['{:X}'.format(x) for x in range(0, 16)] + if not set(os.listdir('.')).issubset(expected): + raise RuntimeError( + "%s does not look like a valid version 1 cache directory" % cache) + config = dict() + else: + with open('config') as conf: + config = json.load(conf) + + if args.get('show', None): + print("Current configuration in '%s':" % cache) + print(json.dumps(config, sort_keys=True, + indent=4, separators=(',', ': '))) + # in case of the show argument, emit some stats as well + file_count = 0 + for _, _, files in os.walk('.'): + file_count += len(files) + if file_count: # skip config file if it exists + file_count -= 1 + print("Cache contains %s files" % file_count) + del args['show'] + + # Find any keys that are not currently set but should be + for key in config_entries: + if key not in config: + if 'implicit' in config_entries[key]: + config[key] = config_entries[key]['implicit'] + else: + config[key] = config_entries[key]['default'] + if key not in args: + args[key] = config_entries[key]['default'] + + # Now go through each entry in args to see if it changes an existing config + # setting. + for key in args: + if args[key] != config[key]: + if 'converter' in config_entries[key]: + config_entries[key]['converter'](config[key], args[key]) + config[key] = args[key] + + # and write the updated config file + with open('config', 'w') as conf: + json.dump(config, conf) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/sconsign.py b/scripts/sconsign.py index 726838c..819b9f7 100644 --- a/scripts/sconsign.py +++ b/scripts/sconsign.py @@ -35,20 +35,16 @@ __date__ = "__DATE__" __developer__ = "__DEVELOPER__" +import getopt import os import sys +from dbm import whichdb -############################################################################## -# BEGIN STANDARD SCons SCRIPT HEADER -# -# This is the cut-and-paste logic so that a self-contained script can -# interoperate correctly with different SCons versions and installation -# locations for the engine. If you modify anything in this section, you -# should also change other scripts that use this same header. -############################################################################## +import time +import pickle -# compatibility check -if sys.version_info < (3,5,0): +# python compatibility check +if sys.version_info < (3, 5, 0): msg = "scons: *** SCons version %s does not run under Python version %s.\n\ Python >= 3.5 is required.\n" sys.stderr.write(msg % (__version__, sys.version.split()[0])) @@ -68,7 +64,7 @@ if "SCONS_LIB_DIR" in os.environ: libs.append(os.environ["SCONS_LIB_DIR"]) # running from source takes 2nd priority (since 2.3.2), following SCONS_LIB_DIR -source_path = os.path.join(script_path, os.pardir, 'engine') +source_path = os.path.join(script_path, os.pardir, 'src', 'engine') if os.path.isdir(source_path): libs.append(source_path) @@ -85,94 +81,6 @@ if os.path.isdir(local): scons_version = 'scons-%s' % __version__ -# preferred order of scons lookup paths -prefs = [] - -# if we can find package information, use it -try: - import pkg_resources -except ImportError: - pass -else: - try: - d = pkg_resources.get_distribution('scons') - except pkg_resources.DistributionNotFound: - pass - else: - prefs.append(d.location) - -if sys.platform == 'win32': - # Use only sys.prefix on Windows - prefs.append(sys.prefix) - prefs.append(os.path.join(sys.prefix, 'Lib', 'site-packages')) -else: - # On other (POSIX) platforms, things are more complicated due to - # the variety of path names and library locations. - # Build up some possibilities, then transform them into candidates - temp = [] - if script_dir == 'bin': - # script_dir is `pwd`/bin; - # check `pwd`/lib/scons*. - temp.append(os.getcwd()) - else: - if script_dir in ('.', ''): - script_dir = os.getcwd() - head, tail = os.path.split(script_dir) - if tail == "bin": - # script_dir is /foo/bin; - # check /foo/lib/scons*. - temp.append(head) - - head, tail = os.path.split(sys.prefix) - if tail == "usr": - # sys.prefix is /foo/usr; - # check /foo/usr/lib/scons* first, - # then /foo/usr/local/lib/scons*. - temp.append(sys.prefix) - temp.append(os.path.join(sys.prefix, "local")) - elif tail == "local": - h, t = os.path.split(head) - if t == "usr": - # sys.prefix is /foo/usr/local; - # check /foo/usr/local/lib/scons* first, - # then /foo/usr/lib/scons*. - temp.append(sys.prefix) - temp.append(head) - else: - # sys.prefix is /foo/local; - # check only /foo/local/lib/scons*. - temp.append(sys.prefix) - else: - # sys.prefix is /foo (ends in neither /usr or /local); - # check only /foo/lib/scons*. - temp.append(sys.prefix) - - # suffix these to add to our original prefs: - prefs.extend([os.path.join(x, 'lib') for x in temp]) - prefs.extend([os.path.join(x, 'lib', 'python' + sys.version[:3], - 'site-packages') for x in temp]) - - - # Add the parent directory of the current python's library to the - # preferences. This picks up differences between, e.g., lib and lib64, - # and finds the base location in case of a non-copying virtualenv. - try: - libpath = os.__file__ - except AttributeError: - pass - else: - # Split /usr/libfoo/python*/os.py to /usr/libfoo/python*. - libpath, _ = os.path.split(libpath) - # Split /usr/libfoo/python* to /usr/libfoo - libpath, tail = os.path.split(libpath) - # Check /usr/libfoo/scons*. - prefs.append(libpath) - -# Look first for 'scons-__version__' in all of our preference libs, -# then for 'scons'. Skip paths that do not exist. -libs.extend([os.path.join(x, scons_version) for x in prefs if os.path.isdir(x)]) -libs.extend([os.path.join(x, 'scons') for x in prefs if os.path.isdir(x)]) - sys.path = libs + sys.path ############################################################################## @@ -180,17 +88,6 @@ sys.path = libs + sys.path ############################################################################## import SCons.compat - -try: - import whichdb - - whichdb = whichdb.whichdb -except ImportError as e: - from dbm import whichdb - -import time -import pickle - import SCons.SConsign @@ -202,16 +99,8 @@ def my_whichdb(filename): return "SCons.dblite" except IOError: pass - return _orig_whichdb(filename) - - -# Should work on python2 -_orig_whichdb = whichdb -whichdb = my_whichdb + return whichdb(filename) -# was changed for python3 -#_orig_whichdb = whichdb.whichdb -#dbm.whichdb = my_whichdb def my_import(mname): import imp @@ -364,8 +253,10 @@ def field(name, entry, verbose=Verbose): def nodeinfo_raw(name, ninfo, prefix=""): - # This just formats the dictionary, which we would normally use str() - # to do, except that we want the keys sorted for deterministic output. + """ + This just formats the dictionary, which we would normally use str() + to do, except that we want the keys sorted for deterministic output. + """ d = ninfo.__getstate__() try: keys = ninfo.field_list + ['_version_id'] @@ -536,54 +427,97 @@ def Do_SConsignDir(name): ############################################################################## +def main(): + global Do_Call + + helpstr = """\ + Usage: sconsign [OPTIONS] [FILE ...] + Options: + -a, --act, --action Print build action information. + -c, --csig Print content signature information. + -d DIR, --dir=DIR Print only info about DIR. + -e ENTRY, --entry=ENTRY Print only info about ENTRY. + -f FORMAT, --format=FORMAT FILE is in the specified FORMAT. + -h, --help Print this message and exit. + -i, --implicit Print implicit dependency information. + -r, --readable Print timestamps in human-readable form. + --raw Print raw Python object representations. + -s, --size Print file sizes. + -t, --timestamp Print timestamp information. + -v, --verbose Verbose, describe each field. + """ -import getopt - -helpstr = """\ -Usage: sconsign [OPTIONS] [FILE ...] -Options: - -a, --act, --action Print build action information. - -c, --csig Print content signature information. - -d DIR, --dir=DIR Print only info about DIR. - -e ENTRY, --entry=ENTRY Print only info about ENTRY. - -f FORMAT, --format=FORMAT FILE is in the specified FORMAT. - -h, --help Print this message and exit. - -i, --implicit Print implicit dependency information. - -r, --readable Print timestamps in human-readable form. - --raw Print raw Python object representations. - -s, --size Print file sizes. - -t, --timestamp Print timestamp information. - -v, --verbose Verbose, describe each field. -""" - -try: - opts, args = getopt.getopt(sys.argv[1:], "acd:e:f:hirstv", - ['act', 'action', - 'csig', 'dir=', 'entry=', - 'format=', 'help', 'implicit', - 'raw', 'readable', - 'size', 'timestamp', 'verbose']) -except getopt.GetoptError as err: - sys.stderr.write(str(err) + '\n') - print(helpstr) - sys.exit(2) - -for o, a in opts: - if o in ('-a', '--act', '--action'): - Print_Flags['action'] = 1 - elif o in ('-c', '--csig'): - Print_Flags['csig'] = 1 - elif o in ('-d', '--dir'): - Print_Directories.append(a) - elif o in ('-e', '--entry'): - Print_Entries.append(a) - elif o in ('-f', '--format'): - # Try to map the given DB format to a known module - # name, that we can then try to import... - Module_Map = {'dblite': 'SCons.dblite', 'sconsign': None} - dbm_name = Module_Map.get(a, a) - if dbm_name: - try: + try: + opts, args = getopt.getopt(sys.argv[1:], "acd:e:f:hirstv", + ['act', 'action', + 'csig', 'dir=', 'entry=', + 'format=', 'help', 'implicit', + 'raw', 'readable', + 'size', 'timestamp', 'verbose']) + except getopt.GetoptError as err: + sys.stderr.write(str(err) + '\n') + print(helpstr) + sys.exit(2) + + for o, a in opts: + if o in ('-a', '--act', '--action'): + Print_Flags['action'] = 1 + elif o in ('-c', '--csig'): + Print_Flags['csig'] = 1 + elif o in ('-d', '--dir'): + Print_Directories.append(a) + elif o in ('-e', '--entry'): + Print_Entries.append(a) + elif o in ('-f', '--format'): + # Try to map the given DB format to a known module + # name, that we can then try to import... + Module_Map = {'dblite': 'SCons.dblite', 'sconsign': None} + dbm_name = Module_Map.get(a, a) + if dbm_name: + try: + if dbm_name != "SCons.dblite": + dbm = my_import(dbm_name) + else: + import SCons.dblite + + dbm = SCons.dblite + # Ensure that we don't ignore corrupt DB files, + # this was handled by calling my_import('SCons.dblite') + # again in earlier versions... + SCons.dblite.ignore_corrupt_dbfiles = 0 + except ImportError: + sys.stderr.write("sconsign: illegal file format `%s'\n" % a) + print(helpstr) + sys.exit(2) + Do_Call = Do_SConsignDB(a, dbm) + else: + Do_Call = Do_SConsignDir + elif o in ('-h', '--help'): + print(helpstr) + sys.exit(0) + elif o in ('-i', '--implicit'): + Print_Flags['implicit'] = 1 + elif o in ('--raw',): + nodeinfo_string = nodeinfo_raw + elif o in ('-r', '--readable'): + Readable = 1 + elif o in ('-s', '--size'): + Print_Flags['size'] = 1 + elif o in ('-t', '--timestamp'): + Print_Flags['timestamp'] = 1 + elif o in ('-v', '--verbose'): + Verbose = 1 + + if Do_Call: + for a in args: + Do_Call(a) + else: + if not args: + args = [".sconsign.dblite"] + for a in args: + dbm_name = my_whichdb(a) + if dbm_name: + Map_Module = {'SCons.dblite': 'dblite'} if dbm_name != "SCons.dblite": dbm = my_import(dbm_name) else: @@ -594,56 +528,18 @@ for o, a in opts: # this was handled by calling my_import('SCons.dblite') # again in earlier versions... SCons.dblite.ignore_corrupt_dbfiles = 0 - except ImportError: - sys.stderr.write("sconsign: illegal file format `%s'\n" % a) - print(helpstr) - sys.exit(2) - Do_Call = Do_SConsignDB(a, dbm) - else: - Do_Call = Do_SConsignDir - elif o in ('-h', '--help'): - print(helpstr) - sys.exit(0) - elif o in ('-i', '--implicit'): - Print_Flags['implicit'] = 1 - elif o in ('--raw',): - nodeinfo_string = nodeinfo_raw - elif o in ('-r', '--readable'): - Readable = 1 - elif o in ('-s', '--size'): - Print_Flags['size'] = 1 - elif o in ('-t', '--timestamp'): - Print_Flags['timestamp'] = 1 - elif o in ('-v', '--verbose'): - Verbose = 1 - -if Do_Call: - for a in args: - Do_Call(a) -else: - if not args: - args = [".sconsign.dblite"] - for a in args: - dbm_name = whichdb(a) - if dbm_name: - Map_Module = {'SCons.dblite': 'dblite'} - if dbm_name != "SCons.dblite": - dbm = my_import(dbm_name) + Do_SConsignDB(Map_Module.get(dbm_name, dbm_name), dbm)(a) else: - import SCons.dblite - - dbm = SCons.dblite - # Ensure that we don't ignore corrupt DB files, - # this was handled by calling my_import('SCons.dblite') - # again in earlier versions... - SCons.dblite.ignore_corrupt_dbfiles = 0 - Do_SConsignDB(Map_Module.get(dbm_name, dbm_name), dbm)(a) - else: - Do_SConsignDir(a) + Do_SConsignDir(a) + + if Warns: + print("NOTE: there were %d warnings, please check output" % Warns) + + - if Warns: - print("NOTE: there were %d warnings, please check output" % Warns) -sys.exit(0) +if __name__ == "__main__": + main() + sys.exit(0) # Local Variables: # tab-width:4 -- cgit v0.12 From cd30b7ee366aa0a45b00b27dff287143dbfe0265 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 21 Mar 2020 17:08:33 -0700 Subject: move sconsign logic into src/engine/SCons/Utilties, update entry_points to generate sconsign script --- setup.cfg | 1 + src/engine/SCons/Utilities/__init__.py | 0 src/engine/SCons/Utilities/sconsign.py | 548 +++++++++++++++++++++++++++++++++ 3 files changed, 549 insertions(+) create mode 100644 src/engine/SCons/Utilities/__init__.py create mode 100644 src/engine/SCons/Utilities/sconsign.py diff --git a/setup.cfg b/setup.cfg index d44c1f5..8e55f10 100644 --- a/setup.cfg +++ b/setup.cfg @@ -53,6 +53,7 @@ packages = find: [options.entry_points] console_scripts = scons = SCons.Script.Main:main + sconsign = SCons.Utilities.sconsign:main [options.package_data] diff --git a/src/engine/SCons/Utilities/__init__.py b/src/engine/SCons/Utilities/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/engine/SCons/Utilities/sconsign.py b/src/engine/SCons/Utilities/sconsign.py new file mode 100644 index 0000000..819b9f7 --- /dev/null +++ b/src/engine/SCons/Utilities/sconsign.py @@ -0,0 +1,548 @@ +#! /usr/bin/env python +# +# SCons - a Software Constructor +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +__version__ = "__VERSION__" + +__build__ = "__BUILD__" + +__buildsys__ = "__BUILDSYS__" + +__date__ = "__DATE__" + +__developer__ = "__DEVELOPER__" + +import getopt +import os +import sys +from dbm import whichdb + +import time +import pickle + +# python compatibility check +if sys.version_info < (3, 5, 0): + msg = "scons: *** SCons version %s does not run under Python version %s.\n\ +Python >= 3.5 is required.\n" + sys.stderr.write(msg % (__version__, sys.version.split()[0])) + sys.exit(1) + +# Strip the script directory from sys.path so on case-insensitive +# (WIN32) systems Python doesn't think that the "scons" script is the +# "SCons" package. +script_dir = os.path.dirname(os.path.realpath(__file__)) +script_path = os.path.realpath(os.path.dirname(__file__)) +if script_path in sys.path: + sys.path.remove(script_path) + +libs = [] + +if "SCONS_LIB_DIR" in os.environ: + libs.append(os.environ["SCONS_LIB_DIR"]) + +# running from source takes 2nd priority (since 2.3.2), following SCONS_LIB_DIR +source_path = os.path.join(script_path, os.pardir, 'src', 'engine') +if os.path.isdir(source_path): + libs.append(source_path) + +# add local-install locations +local_version = 'scons-local-' + __version__ +local = 'scons-local' +if script_dir: + local_version = os.path.join(script_dir, local_version) + local = os.path.join(script_dir, local) +if os.path.isdir(local_version): + libs.append(os.path.abspath(local_version)) +if os.path.isdir(local): + libs.append(os.path.abspath(local)) + +scons_version = 'scons-%s' % __version__ + +sys.path = libs + sys.path + +############################################################################## +# END STANDARD SCons SCRIPT HEADER +############################################################################## + +import SCons.compat +import SCons.SConsign + + +def my_whichdb(filename): + if filename[-7:] == ".dblite": + return "SCons.dblite" + try: + with open(filename + ".dblite", "rb"): + return "SCons.dblite" + except IOError: + pass + return whichdb(filename) + + +def my_import(mname): + import imp + + if '.' in mname: + i = mname.rfind('.') + parent = my_import(mname[:i]) + fp, pathname, description = imp.find_module(mname[i+1:], + parent.__path__) + else: + fp, pathname, description = imp.find_module(mname) + return imp.load_module(mname, fp, pathname, description) + + +class Flagger(object): + default_value = 1 + + def __setitem__(self, item, value): + self.__dict__[item] = value + self.default_value = 0 + + def __getitem__(self, item): + return self.__dict__.get(item, self.default_value) + + +Do_Call = None +Print_Directories = [] +Print_Entries = [] +Print_Flags = Flagger() +Verbose = 0 +Readable = 0 +Warns = 0 + + +def default_mapper(entry, name): + """ + Stringify an entry that doesn't have an explicit mapping. + + Args: + entry: entry + name: field name + + Returns: str + + """ + try: + val = eval("entry." + name) + except AttributeError: + val = None + if sys.version_info.major >= 3 and isinstance(val, bytes): + # This is a dirty hack for py 2/3 compatibility. csig is a bytes object + # in Python3 while Python2 bytes are str. Hence, we decode the csig to a + # Python3 string + val = val.decode() + return str(val) + + +def map_action(entry, _): + """ + Stringify an action entry and signature. + + Args: + entry: action entry + second argument is not used + + Returns: str + + """ + try: + bact = entry.bact + bactsig = entry.bactsig + except AttributeError: + return None + return '%s [%s]' % (bactsig, bact) + + +def map_timestamp(entry, _): + """ + Stringify a timestamp entry. + + Args: + entry: timestamp entry + second argument is not used + + Returns: str + + """ + try: + timestamp = entry.timestamp + except AttributeError: + timestamp = None + if Readable and timestamp: + return "'" + time.ctime(timestamp) + "'" + else: + return str(timestamp) + + +def map_bkids(entry, _): + """ + Stringify an implicit entry. + + Args: + entry: + second argument is not used + + Returns: str + + """ + try: + bkids = entry.bsources + entry.bdepends + entry.bimplicit + bkidsigs = entry.bsourcesigs + entry.bdependsigs + entry.bimplicitsigs + except AttributeError: + return None + + if len(bkids) != len(bkidsigs): + global Warns + Warns += 1 + # add warning to result rather than direct print so it will line up + msg = "Warning: missing information, {} ids but {} sigs" + result = [msg.format(len(bkids), len(bkidsigs))] + else: + result = [] + result += [nodeinfo_string(bkid, bkidsig, " ") + for bkid, bkidsig in zip(bkids, bkidsigs)] + if not result: + return None + return "\n ".join(result) + + +map_field = { + 'action' : map_action, + 'timestamp' : map_timestamp, + 'bkids' : map_bkids, +} + +map_name = { + 'implicit' : 'bkids', +} + + +def field(name, entry, verbose=Verbose): + if not Print_Flags[name]: + return None + fieldname = map_name.get(name, name) + mapper = map_field.get(fieldname, default_mapper) + val = mapper(entry, name) + if verbose: + val = name + ": " + val + return val + + +def nodeinfo_raw(name, ninfo, prefix=""): + """ + This just formats the dictionary, which we would normally use str() + to do, except that we want the keys sorted for deterministic output. + """ + d = ninfo.__getstate__() + try: + keys = ninfo.field_list + ['_version_id'] + except AttributeError: + keys = sorted(d.keys()) + l = [] + for k in keys: + l.append('%s: %s' % (repr(k), repr(d.get(k)))) + if '\n' in name: + name = repr(name) + return name + ': {' + ', '.join(l) + '}' + + +def nodeinfo_cooked(name, ninfo, prefix=""): + try: + field_list = ninfo.field_list + except AttributeError: + field_list = [] + if '\n' in name: + name = repr(name) + outlist = [name + ':'] + [ + f for f in [field(x, ninfo, Verbose) for x in field_list] if f + ] + if Verbose: + sep = '\n ' + prefix + else: + sep = ' ' + return sep.join(outlist) + + +nodeinfo_string = nodeinfo_cooked + + +def printfield(name, entry, prefix=""): + outlist = field("implicit", entry, 0) + if outlist: + if Verbose: + print(" implicit:") + print(" " + outlist) + outact = field("action", entry, 0) + if outact: + if Verbose: + print(" action: " + outact) + else: + print(" " + outact) + + +def printentries(entries, location): + if Print_Entries: + for name in Print_Entries: + try: + entry = entries[name] + except KeyError: + err = "sconsign: no entry `%s' in `%s'\n" % (name, location) + sys.stderr.write(err) + else: + try: + ninfo = entry.ninfo + except AttributeError: + print(name + ":") + else: + print(nodeinfo_string(name, entry.ninfo)) + printfield(name, entry.binfo) + else: + for name in sorted(entries.keys()): + entry = entries[name] + try: + ninfo = entry.ninfo + except AttributeError: + print(name + ":") + else: + print(nodeinfo_string(name, entry.ninfo)) + printfield(name, entry.binfo) + + +class Do_SConsignDB(object): + def __init__(self, dbm_name, dbm): + self.dbm_name = dbm_name + self.dbm = dbm + + def __call__(self, fname): + # The *dbm modules stick their own file suffixes on the names + # that are passed in. This causes us to jump through some + # hoops here. + try: + # Try opening the specified file name. Example: + # SPECIFIED OPENED BY self.dbm.open() + # --------- ------------------------- + # .sconsign => .sconsign.dblite + # .sconsign.dblite => .sconsign.dblite.dblite + db = self.dbm.open(fname, "r") + except (IOError, OSError) as e: + print_e = e + try: + # That didn't work, so try opening the base name, + # so that if they actually passed in 'sconsign.dblite' + # (for example), the dbm module will put the suffix back + # on for us and open it anyway. + db = self.dbm.open(os.path.splitext(fname)[0], "r") + except (IOError, OSError): + # That didn't work either. See if the file name + # they specified even exists (independent of the dbm + # suffix-mangling). + try: + with open(fname, "rb"): + pass # this is a touch only, we don't use it here. + except (IOError, OSError) as e: + # Nope, that file doesn't even exist, so report that + # fact back. + print_e = e + sys.stderr.write("sconsign: %s\n" % print_e) + return + except KeyboardInterrupt: + raise + except pickle.UnpicklingError: + sys.stderr.write("sconsign: ignoring invalid `%s' file `%s'\n" + % (self.dbm_name, fname)) + return + except Exception as e: + sys.stderr.write("sconsign: ignoring invalid `%s' file `%s': %s\n" + % (self.dbm_name, fname, e)) + exc_type, _, _ = sys.exc_info() + if exc_type.__name__ == "ValueError" and sys.version_info < (3,0,0): + sys.stderr.write("Python 2 only supports pickle protocols 0-2.\n") + return + + if Print_Directories: + for dir in Print_Directories: + try: + val = db[dir] + except KeyError: + err = "sconsign: no dir `%s' in `%s'\n" % (dir, args[0]) + sys.stderr.write(err) + else: + self.printentries(dir, val) + else: + for dir in sorted(db.keys()): + self.printentries(dir, db[dir]) + + @staticmethod + def printentries(dir, val): + try: + print('=== ' + dir + ':') + except TypeError: + print('=== ' + dir.decode() + ':') + printentries(pickle.loads(val), dir) + + +def Do_SConsignDir(name): + try: + with open(name, 'rb') as fp: + try: + sconsign = SCons.SConsign.Dir(fp) + except KeyboardInterrupt: + raise + except pickle.UnpicklingError: + err = "sconsign: ignoring invalid .sconsign file `%s'\n" % name + sys.stderr.write(err) + return + except Exception as e: + err = "sconsign: ignoring invalid .sconsign file `%s': %s\n" % (name, e) + sys.stderr.write(err) + return + printentries(sconsign.entries, args[0]) + except (IOError, OSError) as e: + sys.stderr.write("sconsign: %s\n" % e) + return + + +############################################################################## +def main(): + global Do_Call + + helpstr = """\ + Usage: sconsign [OPTIONS] [FILE ...] + Options: + -a, --act, --action Print build action information. + -c, --csig Print content signature information. + -d DIR, --dir=DIR Print only info about DIR. + -e ENTRY, --entry=ENTRY Print only info about ENTRY. + -f FORMAT, --format=FORMAT FILE is in the specified FORMAT. + -h, --help Print this message and exit. + -i, --implicit Print implicit dependency information. + -r, --readable Print timestamps in human-readable form. + --raw Print raw Python object representations. + -s, --size Print file sizes. + -t, --timestamp Print timestamp information. + -v, --verbose Verbose, describe each field. + """ + + try: + opts, args = getopt.getopt(sys.argv[1:], "acd:e:f:hirstv", + ['act', 'action', + 'csig', 'dir=', 'entry=', + 'format=', 'help', 'implicit', + 'raw', 'readable', + 'size', 'timestamp', 'verbose']) + except getopt.GetoptError as err: + sys.stderr.write(str(err) + '\n') + print(helpstr) + sys.exit(2) + + for o, a in opts: + if o in ('-a', '--act', '--action'): + Print_Flags['action'] = 1 + elif o in ('-c', '--csig'): + Print_Flags['csig'] = 1 + elif o in ('-d', '--dir'): + Print_Directories.append(a) + elif o in ('-e', '--entry'): + Print_Entries.append(a) + elif o in ('-f', '--format'): + # Try to map the given DB format to a known module + # name, that we can then try to import... + Module_Map = {'dblite': 'SCons.dblite', 'sconsign': None} + dbm_name = Module_Map.get(a, a) + if dbm_name: + try: + if dbm_name != "SCons.dblite": + dbm = my_import(dbm_name) + else: + import SCons.dblite + + dbm = SCons.dblite + # Ensure that we don't ignore corrupt DB files, + # this was handled by calling my_import('SCons.dblite') + # again in earlier versions... + SCons.dblite.ignore_corrupt_dbfiles = 0 + except ImportError: + sys.stderr.write("sconsign: illegal file format `%s'\n" % a) + print(helpstr) + sys.exit(2) + Do_Call = Do_SConsignDB(a, dbm) + else: + Do_Call = Do_SConsignDir + elif o in ('-h', '--help'): + print(helpstr) + sys.exit(0) + elif o in ('-i', '--implicit'): + Print_Flags['implicit'] = 1 + elif o in ('--raw',): + nodeinfo_string = nodeinfo_raw + elif o in ('-r', '--readable'): + Readable = 1 + elif o in ('-s', '--size'): + Print_Flags['size'] = 1 + elif o in ('-t', '--timestamp'): + Print_Flags['timestamp'] = 1 + elif o in ('-v', '--verbose'): + Verbose = 1 + + if Do_Call: + for a in args: + Do_Call(a) + else: + if not args: + args = [".sconsign.dblite"] + for a in args: + dbm_name = my_whichdb(a) + if dbm_name: + Map_Module = {'SCons.dblite': 'dblite'} + if dbm_name != "SCons.dblite": + dbm = my_import(dbm_name) + else: + import SCons.dblite + + dbm = SCons.dblite + # Ensure that we don't ignore corrupt DB files, + # this was handled by calling my_import('SCons.dblite') + # again in earlier versions... + SCons.dblite.ignore_corrupt_dbfiles = 0 + Do_SConsignDB(Map_Module.get(dbm_name, dbm_name), dbm)(a) + else: + Do_SConsignDir(a) + + if Warns: + print("NOTE: there were %d warnings, please check output" % Warns) + + + +if __name__ == "__main__": + main() + sys.exit(0) + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: -- cgit v0.12 From c5a750381cfb99c69e0d20228b666bfb1b0e0a41 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 21 Mar 2020 17:10:04 -0700 Subject: change scripts/sconsign.py to be shell which uses logic in SCons.Utilities.sconsign's main --- scripts/sconsign.py | 457 +--------------------------------------------------- 1 file changed, 1 insertion(+), 456 deletions(-) diff --git a/scripts/sconsign.py b/scripts/sconsign.py index 819b9f7..493bd40 100644 --- a/scripts/sconsign.py +++ b/scripts/sconsign.py @@ -35,13 +35,8 @@ __date__ = "__DATE__" __developer__ = "__DEVELOPER__" -import getopt import os import sys -from dbm import whichdb - -import time -import pickle # python compatibility check if sys.version_info < (3, 5, 0): @@ -86,457 +81,7 @@ sys.path = libs + sys.path ############################################################################## # END STANDARD SCons SCRIPT HEADER ############################################################################## - -import SCons.compat -import SCons.SConsign - - -def my_whichdb(filename): - if filename[-7:] == ".dblite": - return "SCons.dblite" - try: - with open(filename + ".dblite", "rb"): - return "SCons.dblite" - except IOError: - pass - return whichdb(filename) - - -def my_import(mname): - import imp - - if '.' in mname: - i = mname.rfind('.') - parent = my_import(mname[:i]) - fp, pathname, description = imp.find_module(mname[i+1:], - parent.__path__) - else: - fp, pathname, description = imp.find_module(mname) - return imp.load_module(mname, fp, pathname, description) - - -class Flagger(object): - default_value = 1 - - def __setitem__(self, item, value): - self.__dict__[item] = value - self.default_value = 0 - - def __getitem__(self, item): - return self.__dict__.get(item, self.default_value) - - -Do_Call = None -Print_Directories = [] -Print_Entries = [] -Print_Flags = Flagger() -Verbose = 0 -Readable = 0 -Warns = 0 - - -def default_mapper(entry, name): - """ - Stringify an entry that doesn't have an explicit mapping. - - Args: - entry: entry - name: field name - - Returns: str - - """ - try: - val = eval("entry." + name) - except AttributeError: - val = None - if sys.version_info.major >= 3 and isinstance(val, bytes): - # This is a dirty hack for py 2/3 compatibility. csig is a bytes object - # in Python3 while Python2 bytes are str. Hence, we decode the csig to a - # Python3 string - val = val.decode() - return str(val) - - -def map_action(entry, _): - """ - Stringify an action entry and signature. - - Args: - entry: action entry - second argument is not used - - Returns: str - - """ - try: - bact = entry.bact - bactsig = entry.bactsig - except AttributeError: - return None - return '%s [%s]' % (bactsig, bact) - - -def map_timestamp(entry, _): - """ - Stringify a timestamp entry. - - Args: - entry: timestamp entry - second argument is not used - - Returns: str - - """ - try: - timestamp = entry.timestamp - except AttributeError: - timestamp = None - if Readable and timestamp: - return "'" + time.ctime(timestamp) + "'" - else: - return str(timestamp) - - -def map_bkids(entry, _): - """ - Stringify an implicit entry. - - Args: - entry: - second argument is not used - - Returns: str - - """ - try: - bkids = entry.bsources + entry.bdepends + entry.bimplicit - bkidsigs = entry.bsourcesigs + entry.bdependsigs + entry.bimplicitsigs - except AttributeError: - return None - - if len(bkids) != len(bkidsigs): - global Warns - Warns += 1 - # add warning to result rather than direct print so it will line up - msg = "Warning: missing information, {} ids but {} sigs" - result = [msg.format(len(bkids), len(bkidsigs))] - else: - result = [] - result += [nodeinfo_string(bkid, bkidsig, " ") - for bkid, bkidsig in zip(bkids, bkidsigs)] - if not result: - return None - return "\n ".join(result) - - -map_field = { - 'action' : map_action, - 'timestamp' : map_timestamp, - 'bkids' : map_bkids, -} - -map_name = { - 'implicit' : 'bkids', -} - - -def field(name, entry, verbose=Verbose): - if not Print_Flags[name]: - return None - fieldname = map_name.get(name, name) - mapper = map_field.get(fieldname, default_mapper) - val = mapper(entry, name) - if verbose: - val = name + ": " + val - return val - - -def nodeinfo_raw(name, ninfo, prefix=""): - """ - This just formats the dictionary, which we would normally use str() - to do, except that we want the keys sorted for deterministic output. - """ - d = ninfo.__getstate__() - try: - keys = ninfo.field_list + ['_version_id'] - except AttributeError: - keys = sorted(d.keys()) - l = [] - for k in keys: - l.append('%s: %s' % (repr(k), repr(d.get(k)))) - if '\n' in name: - name = repr(name) - return name + ': {' + ', '.join(l) + '}' - - -def nodeinfo_cooked(name, ninfo, prefix=""): - try: - field_list = ninfo.field_list - except AttributeError: - field_list = [] - if '\n' in name: - name = repr(name) - outlist = [name + ':'] + [ - f for f in [field(x, ninfo, Verbose) for x in field_list] if f - ] - if Verbose: - sep = '\n ' + prefix - else: - sep = ' ' - return sep.join(outlist) - - -nodeinfo_string = nodeinfo_cooked - - -def printfield(name, entry, prefix=""): - outlist = field("implicit", entry, 0) - if outlist: - if Verbose: - print(" implicit:") - print(" " + outlist) - outact = field("action", entry, 0) - if outact: - if Verbose: - print(" action: " + outact) - else: - print(" " + outact) - - -def printentries(entries, location): - if Print_Entries: - for name in Print_Entries: - try: - entry = entries[name] - except KeyError: - err = "sconsign: no entry `%s' in `%s'\n" % (name, location) - sys.stderr.write(err) - else: - try: - ninfo = entry.ninfo - except AttributeError: - print(name + ":") - else: - print(nodeinfo_string(name, entry.ninfo)) - printfield(name, entry.binfo) - else: - for name in sorted(entries.keys()): - entry = entries[name] - try: - ninfo = entry.ninfo - except AttributeError: - print(name + ":") - else: - print(nodeinfo_string(name, entry.ninfo)) - printfield(name, entry.binfo) - - -class Do_SConsignDB(object): - def __init__(self, dbm_name, dbm): - self.dbm_name = dbm_name - self.dbm = dbm - - def __call__(self, fname): - # The *dbm modules stick their own file suffixes on the names - # that are passed in. This causes us to jump through some - # hoops here. - try: - # Try opening the specified file name. Example: - # SPECIFIED OPENED BY self.dbm.open() - # --------- ------------------------- - # .sconsign => .sconsign.dblite - # .sconsign.dblite => .sconsign.dblite.dblite - db = self.dbm.open(fname, "r") - except (IOError, OSError) as e: - print_e = e - try: - # That didn't work, so try opening the base name, - # so that if they actually passed in 'sconsign.dblite' - # (for example), the dbm module will put the suffix back - # on for us and open it anyway. - db = self.dbm.open(os.path.splitext(fname)[0], "r") - except (IOError, OSError): - # That didn't work either. See if the file name - # they specified even exists (independent of the dbm - # suffix-mangling). - try: - with open(fname, "rb"): - pass # this is a touch only, we don't use it here. - except (IOError, OSError) as e: - # Nope, that file doesn't even exist, so report that - # fact back. - print_e = e - sys.stderr.write("sconsign: %s\n" % print_e) - return - except KeyboardInterrupt: - raise - except pickle.UnpicklingError: - sys.stderr.write("sconsign: ignoring invalid `%s' file `%s'\n" - % (self.dbm_name, fname)) - return - except Exception as e: - sys.stderr.write("sconsign: ignoring invalid `%s' file `%s': %s\n" - % (self.dbm_name, fname, e)) - exc_type, _, _ = sys.exc_info() - if exc_type.__name__ == "ValueError" and sys.version_info < (3,0,0): - sys.stderr.write("Python 2 only supports pickle protocols 0-2.\n") - return - - if Print_Directories: - for dir in Print_Directories: - try: - val = db[dir] - except KeyError: - err = "sconsign: no dir `%s' in `%s'\n" % (dir, args[0]) - sys.stderr.write(err) - else: - self.printentries(dir, val) - else: - for dir in sorted(db.keys()): - self.printentries(dir, db[dir]) - - @staticmethod - def printentries(dir, val): - try: - print('=== ' + dir + ':') - except TypeError: - print('=== ' + dir.decode() + ':') - printentries(pickle.loads(val), dir) - - -def Do_SConsignDir(name): - try: - with open(name, 'rb') as fp: - try: - sconsign = SCons.SConsign.Dir(fp) - except KeyboardInterrupt: - raise - except pickle.UnpicklingError: - err = "sconsign: ignoring invalid .sconsign file `%s'\n" % name - sys.stderr.write(err) - return - except Exception as e: - err = "sconsign: ignoring invalid .sconsign file `%s': %s\n" % (name, e) - sys.stderr.write(err) - return - printentries(sconsign.entries, args[0]) - except (IOError, OSError) as e: - sys.stderr.write("sconsign: %s\n" % e) - return - - -############################################################################## -def main(): - global Do_Call - - helpstr = """\ - Usage: sconsign [OPTIONS] [FILE ...] - Options: - -a, --act, --action Print build action information. - -c, --csig Print content signature information. - -d DIR, --dir=DIR Print only info about DIR. - -e ENTRY, --entry=ENTRY Print only info about ENTRY. - -f FORMAT, --format=FORMAT FILE is in the specified FORMAT. - -h, --help Print this message and exit. - -i, --implicit Print implicit dependency information. - -r, --readable Print timestamps in human-readable form. - --raw Print raw Python object representations. - -s, --size Print file sizes. - -t, --timestamp Print timestamp information. - -v, --verbose Verbose, describe each field. - """ - - try: - opts, args = getopt.getopt(sys.argv[1:], "acd:e:f:hirstv", - ['act', 'action', - 'csig', 'dir=', 'entry=', - 'format=', 'help', 'implicit', - 'raw', 'readable', - 'size', 'timestamp', 'verbose']) - except getopt.GetoptError as err: - sys.stderr.write(str(err) + '\n') - print(helpstr) - sys.exit(2) - - for o, a in opts: - if o in ('-a', '--act', '--action'): - Print_Flags['action'] = 1 - elif o in ('-c', '--csig'): - Print_Flags['csig'] = 1 - elif o in ('-d', '--dir'): - Print_Directories.append(a) - elif o in ('-e', '--entry'): - Print_Entries.append(a) - elif o in ('-f', '--format'): - # Try to map the given DB format to a known module - # name, that we can then try to import... - Module_Map = {'dblite': 'SCons.dblite', 'sconsign': None} - dbm_name = Module_Map.get(a, a) - if dbm_name: - try: - if dbm_name != "SCons.dblite": - dbm = my_import(dbm_name) - else: - import SCons.dblite - - dbm = SCons.dblite - # Ensure that we don't ignore corrupt DB files, - # this was handled by calling my_import('SCons.dblite') - # again in earlier versions... - SCons.dblite.ignore_corrupt_dbfiles = 0 - except ImportError: - sys.stderr.write("sconsign: illegal file format `%s'\n" % a) - print(helpstr) - sys.exit(2) - Do_Call = Do_SConsignDB(a, dbm) - else: - Do_Call = Do_SConsignDir - elif o in ('-h', '--help'): - print(helpstr) - sys.exit(0) - elif o in ('-i', '--implicit'): - Print_Flags['implicit'] = 1 - elif o in ('--raw',): - nodeinfo_string = nodeinfo_raw - elif o in ('-r', '--readable'): - Readable = 1 - elif o in ('-s', '--size'): - Print_Flags['size'] = 1 - elif o in ('-t', '--timestamp'): - Print_Flags['timestamp'] = 1 - elif o in ('-v', '--verbose'): - Verbose = 1 - - if Do_Call: - for a in args: - Do_Call(a) - else: - if not args: - args = [".sconsign.dblite"] - for a in args: - dbm_name = my_whichdb(a) - if dbm_name: - Map_Module = {'SCons.dblite': 'dblite'} - if dbm_name != "SCons.dblite": - dbm = my_import(dbm_name) - else: - import SCons.dblite - - dbm = SCons.dblite - # Ensure that we don't ignore corrupt DB files, - # this was handled by calling my_import('SCons.dblite') - # again in earlier versions... - SCons.dblite.ignore_corrupt_dbfiles = 0 - Do_SConsignDB(Map_Module.get(dbm_name, dbm_name), dbm)(a) - else: - Do_SConsignDir(a) - - if Warns: - print("NOTE: there were %d warnings, please check output" % Warns) - - - +from SCons.Utilities.sconsign import main if __name__ == "__main__": main() sys.exit(0) -- cgit v0.12 From 309b955883e72bebca17ce54c4a58e442107c90c Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 21 Mar 2020 17:16:24 -0700 Subject: move scons-time.py from scripts to bin. As far as I can tell this is really for internal use only --- bin/scons-time.py | 1480 +++++++++++++++++++++++++++++++++++++++++++++++++ scripts/scons-time.py | 1480 ------------------------------------------------- 2 files changed, 1480 insertions(+), 1480 deletions(-) create mode 100644 bin/scons-time.py delete mode 100644 scripts/scons-time.py diff --git a/bin/scons-time.py b/bin/scons-time.py new file mode 100644 index 0000000..e4dd863 --- /dev/null +++ b/bin/scons-time.py @@ -0,0 +1,1480 @@ +#!/usr/bin/env python +# +# scons-time - run SCons timings and collect statistics +# +# A script for running a configuration through SCons with a standard +# set of invocations to collect timing and memory statistics and to +# capture the results in a consistent set of output files for display +# and analysis. +# + +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +import getopt +import glob +import os +import re +import shutil +import sys +import tempfile +import time +import subprocess + +def HACK_for_exec(cmd, *args): + """ + For some reason, Python won't allow an exec() within a function + that also declares an internal function (including lambda functions). + This function is a hack that calls exec() in a function with no + internal functions. + """ + if not args: exec(cmd) + elif len(args) == 1: exec(cmd, args[0]) + else: exec(cmd, args[0], args[1]) + +class Plotter(object): + def increment_size(self, largest): + """ + Return the size of each horizontal increment line for a specified + maximum value. This returns a value that will provide somewhere + between 5 and 9 horizontal lines on the graph, on some set of + boundaries that are multiples of 10/100/1000/etc. + """ + i = largest // 5 + if not i: + return largest + multiplier = 1 + while i >= 10: + i = i // 10 + multiplier = multiplier * 10 + return i * multiplier + + def max_graph_value(self, largest): + # Round up to next integer. + largest = int(largest) + 1 + increment = self.increment_size(largest) + return ((largest + increment - 1) // increment) * increment + +class Line(object): + def __init__(self, points, type, title, label, comment, fmt="%s %s"): + self.points = points + self.type = type + self.title = title + self.label = label + self.comment = comment + self.fmt = fmt + + def print_label(self, inx, x, y): + if self.label: + print('set label %s "%s" at %0.1f,%0.1f right' % (inx, self.label, x, y)) + + def plot_string(self): + if self.title: + title_string = 'title "%s"' % self.title + else: + title_string = 'notitle' + return "'-' %s with lines lt %s" % (title_string, self.type) + + def print_points(self, fmt=None): + if fmt is None: + fmt = self.fmt + if self.comment: + print('# %s' % self.comment) + for x, y in self.points: + # If y is None, it usually represents some kind of break + # in the line's index number. We might want to represent + # this some way rather than just drawing the line straight + # between the two points on either side. + if y is not None: + print(fmt % (x, y)) + print('e') + + def get_x_values(self): + return [ p[0] for p in self.points ] + + def get_y_values(self): + return [ p[1] for p in self.points ] + +class Gnuplotter(Plotter): + + def __init__(self, title, key_location): + self.lines = [] + self.title = title + self.key_location = key_location + + def line(self, points, type, title=None, label=None, comment=None, fmt='%s %s'): + if points: + line = Line(points, type, title, label, comment, fmt) + self.lines.append(line) + + def plot_string(self, line): + return line.plot_string() + + def vertical_bar(self, x, type, label, comment): + if self.get_min_x() <= x <= self.get_max_x(): + points = [(x, 0), (x, self.max_graph_value(self.get_max_y()))] + self.line(points, type, label, comment) + + def get_all_x_values(self): + result = [] + for line in self.lines: + result.extend(line.get_x_values()) + return [r for r in result if r is not None] + + def get_all_y_values(self): + result = [] + for line in self.lines: + result.extend(line.get_y_values()) + return [r for r in result if r is not None] + + def get_min_x(self): + try: + return self.min_x + except AttributeError: + try: + self.min_x = min(self.get_all_x_values()) + except ValueError: + self.min_x = 0 + return self.min_x + + def get_max_x(self): + try: + return self.max_x + except AttributeError: + try: + self.max_x = max(self.get_all_x_values()) + except ValueError: + self.max_x = 0 + return self.max_x + + def get_min_y(self): + try: + return self.min_y + except AttributeError: + try: + self.min_y = min(self.get_all_y_values()) + except ValueError: + self.min_y = 0 + return self.min_y + + def get_max_y(self): + try: + return self.max_y + except AttributeError: + try: + self.max_y = max(self.get_all_y_values()) + except ValueError: + self.max_y = 0 + return self.max_y + + def draw(self): + + if not self.lines: + return + + if self.title: + print('set title "%s"' % self.title) + print('set key %s' % self.key_location) + + min_y = self.get_min_y() + max_y = self.max_graph_value(self.get_max_y()) + incr = (max_y - min_y) / 10.0 + start = min_y + (max_y / 2.0) + (2.0 * incr) + position = [ start - (i * incr) for i in range(5) ] + + inx = 1 + for line in self.lines: + line.print_label(inx, line.points[0][0]-1, + position[(inx-1) % len(position)]) + inx += 1 + + plot_strings = [ self.plot_string(l) for l in self.lines ] + print('plot ' + ', \\\n '.join(plot_strings)) + + for line in self.lines: + line.print_points() + + + +def untar(fname): + import tarfile + tar = tarfile.open(name=fname, mode='r') + for tarinfo in tar: + tar.extract(tarinfo) + tar.close() + +def unzip(fname): + import zipfile + zf = zipfile.ZipFile(fname, 'r') + for name in zf.namelist(): + dir = os.path.dirname(name) + try: + os.makedirs(dir) + except: + pass + with open(name, 'wb') as f: + f.write(zf.read(name)) + +def read_tree(dir): + for dirpath, dirnames, filenames in os.walk(dir): + for fn in filenames: + fn = os.path.join(dirpath, fn) + if os.path.isfile(fn): + with open(fn, 'rb') as f: + f.read() + +def redirect_to_file(command, log): + return '%s > %s 2>&1' % (command, log) + +def tee_to_file(command, log): + return '%s 2>&1 | tee %s' % (command, log) + + + +class SConsTimer(object): + """ + Usage: scons-time SUBCOMMAND [ARGUMENTS] + Type "scons-time help SUBCOMMAND" for help on a specific subcommand. + + Available subcommands: + func Extract test-run data for a function + help Provides help + mem Extract --debug=memory data from test runs + obj Extract --debug=count data from test runs + time Extract --debug=time data from test runs + run Runs a test configuration + """ + + name = 'scons-time' + name_spaces = ' '*len(name) + + def makedict(**kw): + return kw + + default_settings = makedict( + chdir = None, + config_file = None, + initial_commands = [], + key_location = 'bottom left', + orig_cwd = os.getcwd(), + outdir = None, + prefix = '', + python = '"%s"' % sys.executable, + redirect = redirect_to_file, + scons = None, + scons_flags = '--debug=count --debug=memory --debug=time --debug=memoizer', + scons_lib_dir = None, + scons_wrapper = None, + startup_targets = '--help', + subdir = None, + subversion_url = None, + svn = 'svn', + svn_co_flag = '-q', + tar = 'tar', + targets = '', + targets0 = None, + targets1 = None, + targets2 = None, + title = None, + unzip = 'unzip', + verbose = False, + vertical_bars = [], + + unpack_map = { + '.tar.gz' : (untar, '%(tar)s xzf %%s'), + '.tgz' : (untar, '%(tar)s xzf %%s'), + '.tar' : (untar, '%(tar)s xf %%s'), + '.zip' : (unzip, '%(unzip)s %%s'), + }, + ) + + run_titles = [ + 'Startup', + 'Full build', + 'Up-to-date build', + ] + + run_commands = [ + '%(python)s %(scons_wrapper)s %(scons_flags)s --profile=%(prof0)s %(targets0)s', + '%(python)s %(scons_wrapper)s %(scons_flags)s --profile=%(prof1)s %(targets1)s', + '%(python)s %(scons_wrapper)s %(scons_flags)s --profile=%(prof2)s %(targets2)s', + ] + + stages = [ + 'pre-read', + 'post-read', + 'pre-build', + 'post-build', + ] + + stage_strings = { + 'pre-read' : 'Memory before reading SConscript files:', + 'post-read' : 'Memory after reading SConscript files:', + 'pre-build' : 'Memory before building targets:', + 'post-build' : 'Memory after building targets:', + } + + memory_string_all = 'Memory ' + + default_stage = stages[-1] + + time_strings = { + 'total' : 'Total build time', + 'SConscripts' : 'Total SConscript file execution time', + 'SCons' : 'Total SCons execution time', + 'commands' : 'Total command execution time', + } + + time_string_all = 'Total .* time' + + # + + def __init__(self): + self.__dict__.update(self.default_settings) + + # Functions for displaying and executing commands. + + def subst(self, x, dictionary): + try: + return x % dictionary + except TypeError: + # x isn't a string (it's probably a Python function), + # so just return it. + return x + + def subst_variables(self, command, dictionary): + """ + Substitutes (via the format operator) the values in the specified + dictionary into the specified command. + + The command can be an (action, string) tuple. In all cases, we + perform substitution on strings and don't worry if something isn't + a string. (It's probably a Python function to be executed.) + """ + try: + command + '' + except TypeError: + action = command[0] + string = command[1] + args = command[2:] + else: + action = command + string = action + args = (()) + action = self.subst(action, dictionary) + string = self.subst(string, dictionary) + return (action, string, args) + + def _do_not_display(self, msg, *args): + pass + + def display(self, msg, *args): + """ + Displays the specified message. + + Each message is prepended with a standard prefix of our name + plus the time. + """ + if callable(msg): + msg = msg(*args) + else: + msg = msg % args + if msg is None: + return + fmt = '%s[%s]: %s\n' + sys.stdout.write(fmt % (self.name, time.strftime('%H:%M:%S'), msg)) + + def _do_not_execute(self, action, *args): + pass + + def execute(self, action, *args): + """ + Executes the specified action. + + The action is called if it's a callable Python function, and + otherwise passed to os.system(). + """ + if callable(action): + action(*args) + else: + os.system(action % args) + + def run_command_list(self, commands, dict): + """ + Executes a list of commands, substituting values from the + specified dictionary. + """ + commands = [ self.subst_variables(c, dict) for c in commands ] + for action, string, args in commands: + self.display(string, *args) + sys.stdout.flush() + status = self.execute(action, *args) + if status: + sys.exit(status) + + def log_display(self, command, log): + command = self.subst(command, self.__dict__) + if log: + command = self.redirect(command, log) + return command + + def log_execute(self, command, log): + command = self.subst(command, self.__dict__) + p = os.popen(command) + output = p.read() + p.close() + #TODO: convert to subrocess, os.popen is obsolete. This didn't work: + #process = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True) + #output = process.stdout.read() + #process.stdout.close() + #process.wait() + if self.verbose: + sys.stdout.write(output) + # TODO: Figure out + # Not sure we need to write binary here + with open(log, 'w') as f: + f.write(str(output)) + + def archive_splitext(self, path): + """ + Splits an archive name into a filename base and extension. + + This is like os.path.splitext() (which it calls) except that it + also looks for '.tar.gz' and treats it as an atomic extensions. + """ + if path.endswith('.tar.gz'): + return path[:-7], path[-7:] + else: + return os.path.splitext(path) + + def args_to_files(self, args, tail=None): + """ + Takes a list of arguments, expands any glob patterns, and + returns the last "tail" files from the list. + """ + files = [] + for a in args: + files.extend(sorted(glob.glob(a))) + + if tail: + files = files[-tail:] + + return files + + def ascii_table(self, files, columns, + line_function, file_function=lambda x: x, + *args, **kw): + + header_fmt = ' '.join(['%12s'] * len(columns)) + line_fmt = header_fmt + ' %s' + + print(header_fmt % columns) + + for file in files: + t = line_function(file, *args, **kw) + if t is None: + t = [] + diff = len(columns) - len(t) + if diff > 0: + t += [''] * diff + t.append(file_function(file)) + print(line_fmt % tuple(t)) + + def collect_results(self, files, function, *args, **kw): + results = {} + + for file in files: + base = os.path.splitext(file)[0] + run, index = base.split('-')[-2:] + + run = int(run) + index = int(index) + + value = function(file, *args, **kw) + + try: + r = results[index] + except KeyError: + r = [] + results[index] = r + r.append((run, value)) + + return results + + def doc_to_help(self, obj): + """ + Translates an object's __doc__ string into help text. + + This strips a consistent number of spaces from each line in the + help text, essentially "outdenting" the text to the left-most + column. + """ + doc = obj.__doc__ + if doc is None: + return '' + return self.outdent(doc) + + def find_next_run_number(self, dir, prefix): + """ + Returns the next run number in a directory for the specified prefix. + + Examines the contents the specified directory for files with the + specified prefix, extracts the run numbers from each file name, + and returns the next run number after the largest it finds. + """ + x = re.compile(re.escape(prefix) + '-([0-9]+).*') + matches = [x.match(e) for e in os.listdir(dir)] + matches = [_f for _f in matches if _f] + if not matches: + return 0 + run_numbers = [int(m.group(1)) for m in matches] + return int(max(run_numbers)) + 1 + + def gnuplot_results(self, results, fmt='%s %.3f'): + """ + Prints out a set of results in Gnuplot format. + """ + gp = Gnuplotter(self.title, self.key_location) + + for i in sorted(results.keys()): + try: + t = self.run_titles[i] + except IndexError: + t = '??? %s ???' % i + results[i].sort() + gp.line(results[i], i+1, t, None, t, fmt=fmt) + + for bar_tuple in self.vertical_bars: + try: + x, type, label, comment = bar_tuple + except ValueError: + x, type, label = bar_tuple + comment = label + gp.vertical_bar(x, type, label, comment) + + gp.draw() + + def logfile_name(self, invocation): + """ + Returns the absolute path of a log file for the specificed + invocation number. + """ + name = self.prefix_run + '-%d.log' % invocation + return os.path.join(self.outdir, name) + + def outdent(self, s): + """ + Strip as many spaces from each line as are found at the beginning + of the first line in the list. + """ + lines = s.split('\n') + if lines[0] == '': + lines = lines[1:] + spaces = re.match(' *', lines[0]).group(0) + def strip_initial_spaces(l, s=spaces): + if l.startswith(spaces): + l = l[len(spaces):] + return l + return '\n'.join([ strip_initial_spaces(l) for l in lines ]) + '\n' + + def profile_name(self, invocation): + """ + Returns the absolute path of a profile file for the specified + invocation number. + """ + name = self.prefix_run + '-%d.prof' % invocation + return os.path.join(self.outdir, name) + + def set_env(self, key, value): + os.environ[key] = value + + # + + def get_debug_times(self, file, time_string=None): + """ + Fetch times from the --debug=time strings in the specified file. + """ + if time_string is None: + search_string = self.time_string_all + else: + search_string = time_string + with open(file) as f: + contents = f.read() + if not contents: + sys.stderr.write('file %s has no contents!\n' % repr(file)) + return None + result = re.findall(r'%s: ([\d.]*)' % search_string, contents)[-4:] + result = [ float(r) for r in result ] + if time_string is not None: + try: + result = result[0] + except IndexError: + sys.stderr.write('file %s has no results!\n' % repr(file)) + return None + return result + + def get_function_profile(self, file, function): + """ + Returns the file, line number, function name, and cumulative time. + """ + try: + import pstats + except ImportError as e: + sys.stderr.write('%s: func: %s\n' % (self.name, e)) + sys.stderr.write('%s This version of Python is missing the profiler.\n' % self.name_spaces) + sys.stderr.write('%s Cannot use the "func" subcommand.\n' % self.name_spaces) + sys.exit(1) + statistics = pstats.Stats(file).stats + matches = [ e for e in statistics.items() if e[0][2] == function ] + r = matches[0] + return r[0][0], r[0][1], r[0][2], r[1][3] + + def get_function_time(self, file, function): + """ + Returns just the cumulative time for the specified function. + """ + return self.get_function_profile(file, function)[3] + + def get_memory(self, file, memory_string=None): + """ + Returns a list of integers of the amount of memory used. The + default behavior is to return all the stages. + """ + if memory_string is None: + search_string = self.memory_string_all + else: + search_string = memory_string + with open(file) as f: + lines = f.readlines() + lines = [ l for l in lines if l.startswith(search_string) ][-4:] + result = [ int(l.split()[-1]) for l in lines[-4:] ] + if len(result) == 1: + result = result[0] + return result + + def get_object_counts(self, file, object_name, index=None): + """ + Returns the counts of the specified object_name. + """ + object_string = ' ' + object_name + '\n' + with open(file) as f: + lines = f.readlines() + line = [ l for l in lines if l.endswith(object_string) ][0] + result = [ int(field) for field in line.split()[:4] ] + if index is not None: + result = result[index] + return result + + + command_alias = {} + + def execute_subcommand(self, argv): + """ + Executes the do_*() function for the specified subcommand (argv[0]). + """ + if not argv: + return + cmdName = self.command_alias.get(argv[0], argv[0]) + try: + func = getattr(self, 'do_' + cmdName) + except AttributeError: + return self.default(argv) + try: + return func(argv) + except TypeError as e: + sys.stderr.write("%s %s: %s\n" % (self.name, cmdName, e)) + import traceback + traceback.print_exc(file=sys.stderr) + sys.stderr.write("Try '%s help %s'\n" % (self.name, cmdName)) + + def default(self, argv): + """ + The default behavior for an unknown subcommand. Prints an + error message and exits. + """ + sys.stderr.write('%s: Unknown subcommand "%s".\n' % (self.name, argv[0])) + sys.stderr.write('Type "%s help" for usage.\n' % self.name) + sys.exit(1) + + # + + def do_help(self, argv): + """ + """ + if argv[1:]: + for arg in argv[1:]: + try: + func = getattr(self, 'do_' + arg) + except AttributeError: + sys.stderr.write('%s: No help for "%s"\n' % (self.name, arg)) + else: + try: + help = getattr(self, 'help_' + arg) + except AttributeError: + sys.stdout.write(self.doc_to_help(func)) + sys.stdout.flush() + else: + help() + else: + doc = self.doc_to_help(self.__class__) + if doc: + sys.stdout.write(doc) + sys.stdout.flush() + return None + + # + + def help_func(self): + help = """\ + Usage: scons-time func [OPTIONS] FILE [...] + + -C DIR, --chdir=DIR Change to DIR before looking for files + -f FILE, --file=FILE Read configuration from specified FILE + --fmt=FORMAT, --format=FORMAT Print data in specified FORMAT + --func=NAME, --function=NAME Report time for function NAME + -h, --help Print this help and exit + -p STRING, --prefix=STRING Use STRING as log file/profile prefix + -t NUMBER, --tail=NUMBER Only report the last NUMBER files + --title=TITLE Specify the output plot TITLE + """ + sys.stdout.write(self.outdent(help)) + sys.stdout.flush() + + def do_func(self, argv): + """ + """ + format = 'ascii' + function_name = '_main' + tail = None + + short_opts = '?C:f:hp:t:' + + long_opts = [ + 'chdir=', + 'file=', + 'fmt=', + 'format=', + 'func=', + 'function=', + 'help', + 'prefix=', + 'tail=', + 'title=', + ] + + opts, args = getopt.getopt(argv[1:], short_opts, long_opts) + + for o, a in opts: + if o in ('-C', '--chdir'): + self.chdir = a + elif o in ('-f', '--file'): + self.config_file = a + elif o in ('--fmt', '--format'): + format = a + elif o in ('--func', '--function'): + function_name = a + elif o in ('-?', '-h', '--help'): + self.do_help(['help', 'func']) + sys.exit(0) + elif o in ('--max',): + max_time = int(a) + elif o in ('-p', '--prefix'): + self.prefix = a + elif o in ('-t', '--tail'): + tail = int(a) + elif o in ('--title',): + self.title = a + + if self.config_file: + with open(self.config_file, 'r') as f: + config = f.read() + exec(config, self.__dict__) + + if self.chdir: + os.chdir(self.chdir) + + if not args: + + pattern = '%s*.prof' % self.prefix + args = self.args_to_files([pattern], tail) + + if not args: + if self.chdir: + directory = self.chdir + else: + directory = os.getcwd() + + sys.stderr.write('%s: func: No arguments specified.\n' % self.name) + sys.stderr.write('%s No %s*.prof files found in "%s".\n' % (self.name_spaces, self.prefix, directory)) + sys.stderr.write('%s Type "%s help func" for help.\n' % (self.name_spaces, self.name)) + sys.exit(1) + + else: + + args = self.args_to_files(args, tail) + + cwd_ = os.getcwd() + os.sep + + if format == 'ascii': + + for file in args: + try: + f, line, func, time = \ + self.get_function_profile(file, function_name) + except ValueError as e: + sys.stderr.write("%s: func: %s: %s\n" % + (self.name, file, e)) + else: + if f.startswith(cwd_): + f = f[len(cwd_):] + print("%.3f %s:%d(%s)" % (time, f, line, func)) + + elif format == 'gnuplot': + + results = self.collect_results(args, self.get_function_time, + function_name) + + self.gnuplot_results(results) + + else: + + sys.stderr.write('%s: func: Unknown format "%s".\n' % (self.name, format)) + sys.exit(1) + + # + + def help_mem(self): + help = """\ + Usage: scons-time mem [OPTIONS] FILE [...] + + -C DIR, --chdir=DIR Change to DIR before looking for files + -f FILE, --file=FILE Read configuration from specified FILE + --fmt=FORMAT, --format=FORMAT Print data in specified FORMAT + -h, --help Print this help and exit + -p STRING, --prefix=STRING Use STRING as log file/profile prefix + --stage=STAGE Plot memory at the specified stage: + pre-read, post-read, pre-build, + post-build (default: post-build) + -t NUMBER, --tail=NUMBER Only report the last NUMBER files + --title=TITLE Specify the output plot TITLE + """ + sys.stdout.write(self.outdent(help)) + sys.stdout.flush() + + def do_mem(self, argv): + + format = 'ascii' + logfile_path = lambda x: x + stage = self.default_stage + tail = None + + short_opts = '?C:f:hp:t:' + + long_opts = [ + 'chdir=', + 'file=', + 'fmt=', + 'format=', + 'help', + 'prefix=', + 'stage=', + 'tail=', + 'title=', + ] + + opts, args = getopt.getopt(argv[1:], short_opts, long_opts) + + for o, a in opts: + if o in ('-C', '--chdir'): + self.chdir = a + elif o in ('-f', '--file'): + self.config_file = a + elif o in ('--fmt', '--format'): + format = a + elif o in ('-?', '-h', '--help'): + self.do_help(['help', 'mem']) + sys.exit(0) + elif o in ('-p', '--prefix'): + self.prefix = a + elif o in ('--stage',): + if a not in self.stages: + sys.stderr.write('%s: mem: Unrecognized stage "%s".\n' % (self.name, a)) + sys.exit(1) + stage = a + elif o in ('-t', '--tail'): + tail = int(a) + elif o in ('--title',): + self.title = a + + if self.config_file: + with open(self.config_file, 'r') as f: + config = f.read() + HACK_for_exec(config, self.__dict__) + + if self.chdir: + os.chdir(self.chdir) + logfile_path = lambda x: os.path.join(self.chdir, x) + + if not args: + + pattern = '%s*.log' % self.prefix + args = self.args_to_files([pattern], tail) + + if not args: + if self.chdir: + directory = self.chdir + else: + directory = os.getcwd() + + sys.stderr.write('%s: mem: No arguments specified.\n' % self.name) + sys.stderr.write('%s No %s*.log files found in "%s".\n' % (self.name_spaces, self.prefix, directory)) + sys.stderr.write('%s Type "%s help mem" for help.\n' % (self.name_spaces, self.name)) + sys.exit(1) + + else: + + args = self.args_to_files(args, tail) + + cwd_ = os.getcwd() + os.sep + + if format == 'ascii': + + self.ascii_table(args, tuple(self.stages), self.get_memory, logfile_path) + + elif format == 'gnuplot': + + results = self.collect_results(args, self.get_memory, + self.stage_strings[stage]) + + self.gnuplot_results(results) + + else: + + sys.stderr.write('%s: mem: Unknown format "%s".\n' % (self.name, format)) + sys.exit(1) + + return 0 + + # + + def help_obj(self): + help = """\ + Usage: scons-time obj [OPTIONS] OBJECT FILE [...] + + -C DIR, --chdir=DIR Change to DIR before looking for files + -f FILE, --file=FILE Read configuration from specified FILE + --fmt=FORMAT, --format=FORMAT Print data in specified FORMAT + -h, --help Print this help and exit + -p STRING, --prefix=STRING Use STRING as log file/profile prefix + --stage=STAGE Plot memory at the specified stage: + pre-read, post-read, pre-build, + post-build (default: post-build) + -t NUMBER, --tail=NUMBER Only report the last NUMBER files + --title=TITLE Specify the output plot TITLE + """ + sys.stdout.write(self.outdent(help)) + sys.stdout.flush() + + def do_obj(self, argv): + + format = 'ascii' + logfile_path = lambda x: x + stage = self.default_stage + tail = None + + short_opts = '?C:f:hp:t:' + + long_opts = [ + 'chdir=', + 'file=', + 'fmt=', + 'format=', + 'help', + 'prefix=', + 'stage=', + 'tail=', + 'title=', + ] + + opts, args = getopt.getopt(argv[1:], short_opts, long_opts) + + for o, a in opts: + if o in ('-C', '--chdir'): + self.chdir = a + elif o in ('-f', '--file'): + self.config_file = a + elif o in ('--fmt', '--format'): + format = a + elif o in ('-?', '-h', '--help'): + self.do_help(['help', 'obj']) + sys.exit(0) + elif o in ('-p', '--prefix'): + self.prefix = a + elif o in ('--stage',): + if a not in self.stages: + sys.stderr.write('%s: obj: Unrecognized stage "%s".\n' % (self.name, a)) + sys.stderr.write('%s Type "%s help obj" for help.\n' % (self.name_spaces, self.name)) + sys.exit(1) + stage = a + elif o in ('-t', '--tail'): + tail = int(a) + elif o in ('--title',): + self.title = a + + if not args: + sys.stderr.write('%s: obj: Must specify an object name.\n' % self.name) + sys.stderr.write('%s Type "%s help obj" for help.\n' % (self.name_spaces, self.name)) + sys.exit(1) + + object_name = args.pop(0) + + if self.config_file: + with open(self.config_file, 'r') as f: + config = f.read() + HACK_for_exec(config, self.__dict__) + + if self.chdir: + os.chdir(self.chdir) + logfile_path = lambda x: os.path.join(self.chdir, x) + + if not args: + + pattern = '%s*.log' % self.prefix + args = self.args_to_files([pattern], tail) + + if not args: + if self.chdir: + directory = self.chdir + else: + directory = os.getcwd() + + sys.stderr.write('%s: obj: No arguments specified.\n' % self.name) + sys.stderr.write('%s No %s*.log files found in "%s".\n' % (self.name_spaces, self.prefix, directory)) + sys.stderr.write('%s Type "%s help obj" for help.\n' % (self.name_spaces, self.name)) + sys.exit(1) + + else: + + args = self.args_to_files(args, tail) + + cwd_ = os.getcwd() + os.sep + + if format == 'ascii': + + self.ascii_table(args, tuple(self.stages), self.get_object_counts, logfile_path, object_name) + + elif format == 'gnuplot': + + stage_index = 0 + for s in self.stages: + if stage == s: + break + stage_index = stage_index + 1 + + results = self.collect_results(args, self.get_object_counts, + object_name, stage_index) + + self.gnuplot_results(results) + + else: + + sys.stderr.write('%s: obj: Unknown format "%s".\n' % (self.name, format)) + sys.exit(1) + + return 0 + + # + + def help_run(self): + help = """\ + Usage: scons-time run [OPTIONS] [FILE ...] + + --chdir=DIR Name of unpacked directory for chdir + -f FILE, --file=FILE Read configuration from specified FILE + -h, --help Print this help and exit + -n, --no-exec No execute, just print command lines + --number=NUMBER Put output in files for run NUMBER + --outdir=OUTDIR Put output files in OUTDIR + -p STRING, --prefix=STRING Use STRING as log file/profile prefix + --python=PYTHON Time using the specified PYTHON + -q, --quiet Don't print command lines + --scons=SCONS Time using the specified SCONS + --svn=URL, --subversion=URL Use SCons from Subversion URL + -v, --verbose Display output of commands + """ + sys.stdout.write(self.outdent(help)) + sys.stdout.flush() + + def do_run(self, argv): + """ + """ + run_number_list = [None] + + short_opts = '?f:hnp:qs:v' + + long_opts = [ + 'file=', + 'help', + 'no-exec', + 'number=', + 'outdir=', + 'prefix=', + 'python=', + 'quiet', + 'scons=', + 'svn=', + 'subdir=', + 'subversion=', + 'verbose', + ] + + opts, args = getopt.getopt(argv[1:], short_opts, long_opts) + + for o, a in opts: + if o in ('-f', '--file'): + self.config_file = a + elif o in ('-?', '-h', '--help'): + self.do_help(['help', 'run']) + sys.exit(0) + elif o in ('-n', '--no-exec'): + self.execute = self._do_not_execute + elif o in ('--number',): + run_number_list = self.split_run_numbers(a) + elif o in ('--outdir',): + self.outdir = a + elif o in ('-p', '--prefix'): + self.prefix = a + elif o in ('--python',): + self.python = a + elif o in ('-q', '--quiet'): + self.display = self._do_not_display + elif o in ('-s', '--subdir'): + self.subdir = a + elif o in ('--scons',): + self.scons = a + elif o in ('--svn', '--subversion'): + self.subversion_url = a + elif o in ('-v', '--verbose'): + self.redirect = tee_to_file + self.verbose = True + self.svn_co_flag = '' + + if not args and not self.config_file: + sys.stderr.write('%s: run: No arguments or -f config file specified.\n' % self.name) + sys.stderr.write('%s Type "%s help run" for help.\n' % (self.name_spaces, self.name)) + sys.exit(1) + + if self.config_file: + with open(self.config_file, 'r') as f: + config = f.read() + exec(config, self.__dict__) + + if args: + self.archive_list = args + + archive_file_name = os.path.split(self.archive_list[0])[1] + + if not self.subdir: + self.subdir = self.archive_splitext(archive_file_name)[0] + + if not self.prefix: + self.prefix = self.archive_splitext(archive_file_name)[0] + + prepare = None + if self.subversion_url: + prepare = self.prep_subversion_run + + for run_number in run_number_list: + self.individual_run(run_number, self.archive_list, prepare) + + def split_run_numbers(self, s): + result = [] + for n in s.split(','): + try: + x, y = n.split('-') + except ValueError: + result.append(int(n)) + else: + result.extend(list(range(int(x), int(y)+1))) + return result + + def scons_path(self, dir): + return os.path.join(dir, 'src', 'script', 'scons.py') + + def scons_lib_dir_path(self, dir): + return os.path.join(dir, 'src', 'engine') + + def prep_subversion_run(self, commands, removals): + self.svn_tmpdir = tempfile.mkdtemp(prefix=self.name + '-svn-') + removals.append((shutil.rmtree, 'rm -rf %%s', self.svn_tmpdir)) + + self.scons = self.scons_path(self.svn_tmpdir) + self.scons_lib_dir = self.scons_lib_dir_path(self.svn_tmpdir) + + commands.extend([ + '%(svn)s co %(svn_co_flag)s -r %(run_number)s %(subversion_url)s %(svn_tmpdir)s', + ]) + + def individual_run(self, run_number, archive_list, prepare=None): + """ + Performs an individual run of the default SCons invocations. + """ + + commands = [] + removals = [] + + if prepare: + prepare(commands, removals) + + save_scons = self.scons + save_scons_wrapper = self.scons_wrapper + save_scons_lib_dir = self.scons_lib_dir + + if self.outdir is None: + self.outdir = self.orig_cwd + elif not os.path.isabs(self.outdir): + self.outdir = os.path.join(self.orig_cwd, self.outdir) + + if self.scons is None: + self.scons = self.scons_path(self.orig_cwd) + + if self.scons_lib_dir is None: + self.scons_lib_dir = self.scons_lib_dir_path(self.orig_cwd) + + if self.scons_wrapper is None: + self.scons_wrapper = self.scons + + if not run_number: + run_number = self.find_next_run_number(self.outdir, self.prefix) + + self.run_number = str(run_number) + + self.prefix_run = self.prefix + '-%03d' % run_number + + if self.targets0 is None: + self.targets0 = self.startup_targets + if self.targets1 is None: + self.targets1 = self.targets + if self.targets2 is None: + self.targets2 = self.targets + + self.tmpdir = tempfile.mkdtemp(prefix=self.name + '-') + + commands.extend([ + (os.chdir, 'cd %%s', self.tmpdir), + ]) + + for archive in archive_list: + if not os.path.isabs(archive): + archive = os.path.join(self.orig_cwd, archive) + if os.path.isdir(archive): + dest = os.path.split(archive)[1] + commands.append((shutil.copytree, 'cp -r %%s %%s', archive, dest)) + else: + suffix = self.archive_splitext(archive)[1] + unpack_command = self.unpack_map.get(suffix) + if not unpack_command: + dest = os.path.split(archive)[1] + commands.append((shutil.copyfile, 'cp %%s %%s', archive, dest)) + else: + commands.append(unpack_command + (archive,)) + + commands.extend([ + (os.chdir, 'cd %%s', self.subdir), + ]) + + commands.extend(self.initial_commands) + + commands.extend([ + (lambda: read_tree('.'), + 'find * -type f | xargs cat > /dev/null'), + + (self.set_env, 'export %%s=%%s', + 'SCONS_LIB_DIR', self.scons_lib_dir), + + '%(python)s %(scons_wrapper)s --version', + ]) + + index = 0 + for run_command in self.run_commands: + setattr(self, 'prof%d' % index, self.profile_name(index)) + c = ( + self.log_execute, + self.log_display, + run_command, + self.logfile_name(index), + ) + commands.append(c) + index = index + 1 + + commands.extend([ + (os.chdir, 'cd %%s', self.orig_cwd), + ]) + + if not os.environ.get('PRESERVE'): + commands.extend(removals) + commands.append((shutil.rmtree, 'rm -rf %%s', self.tmpdir)) + + self.run_command_list(commands, self.__dict__) + + self.scons = save_scons + self.scons_lib_dir = save_scons_lib_dir + self.scons_wrapper = save_scons_wrapper + + # + + def help_time(self): + help = """\ + Usage: scons-time time [OPTIONS] FILE [...] + + -C DIR, --chdir=DIR Change to DIR before looking for files + -f FILE, --file=FILE Read configuration from specified FILE + --fmt=FORMAT, --format=FORMAT Print data in specified FORMAT + -h, --help Print this help and exit + -p STRING, --prefix=STRING Use STRING as log file/profile prefix + -t NUMBER, --tail=NUMBER Only report the last NUMBER files + --which=TIMER Plot timings for TIMER: total, + SConscripts, SCons, commands. + """ + sys.stdout.write(self.outdent(help)) + sys.stdout.flush() + + def do_time(self, argv): + + format = 'ascii' + logfile_path = lambda x: x + tail = None + which = 'total' + + short_opts = '?C:f:hp:t:' + + long_opts = [ + 'chdir=', + 'file=', + 'fmt=', + 'format=', + 'help', + 'prefix=', + 'tail=', + 'title=', + 'which=', + ] + + opts, args = getopt.getopt(argv[1:], short_opts, long_opts) + + for o, a in opts: + if o in ('-C', '--chdir'): + self.chdir = a + elif o in ('-f', '--file'): + self.config_file = a + elif o in ('--fmt', '--format'): + format = a + elif o in ('-?', '-h', '--help'): + self.do_help(['help', 'time']) + sys.exit(0) + elif o in ('-p', '--prefix'): + self.prefix = a + elif o in ('-t', '--tail'): + tail = int(a) + elif o in ('--title',): + self.title = a + elif o in ('--which',): + if a not in list(self.time_strings.keys()): + sys.stderr.write('%s: time: Unrecognized timer "%s".\n' % (self.name, a)) + sys.stderr.write('%s Type "%s help time" for help.\n' % (self.name_spaces, self.name)) + sys.exit(1) + which = a + + if self.config_file: + with open(self.config_file, 'r') as f: + config = f.read() + HACK_for_exec(config, self.__dict__) + + if self.chdir: + os.chdir(self.chdir) + logfile_path = lambda x: os.path.join(self.chdir, x) + + if not args: + + pattern = '%s*.log' % self.prefix + args = self.args_to_files([pattern], tail) + + if not args: + if self.chdir: + directory = self.chdir + else: + directory = os.getcwd() + + sys.stderr.write('%s: time: No arguments specified.\n' % self.name) + sys.stderr.write('%s No %s*.log files found in "%s".\n' % (self.name_spaces, self.prefix, directory)) + sys.stderr.write('%s Type "%s help time" for help.\n' % (self.name_spaces, self.name)) + sys.exit(1) + + else: + + args = self.args_to_files(args, tail) + + cwd_ = os.getcwd() + os.sep + + if format == 'ascii': + + columns = ("Total", "SConscripts", "SCons", "commands") + self.ascii_table(args, columns, self.get_debug_times, logfile_path) + + elif format == 'gnuplot': + + results = self.collect_results(args, self.get_debug_times, + self.time_strings[which]) + + self.gnuplot_results(results, fmt='%s %.6f') + + else: + + sys.stderr.write('%s: time: Unknown format "%s".\n' % (self.name, format)) + sys.exit(1) + +if __name__ == '__main__': + opts, args = getopt.getopt(sys.argv[1:], 'h?V', ['help', 'version']) + + ST = SConsTimer() + + for o, a in opts: + if o in ('-?', '-h', '--help'): + ST.do_help(['help']) + sys.exit(0) + elif o in ('-V', '--version'): + sys.stdout.write('scons-time version\n') + sys.exit(0) + + if not args: + sys.stderr.write('Type "%s help" for usage.\n' % ST.name) + sys.exit(1) + + ST.execute_subcommand(args) + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/scripts/scons-time.py b/scripts/scons-time.py deleted file mode 100644 index e4dd863..0000000 --- a/scripts/scons-time.py +++ /dev/null @@ -1,1480 +0,0 @@ -#!/usr/bin/env python -# -# scons-time - run SCons timings and collect statistics -# -# A script for running a configuration through SCons with a standard -# set of invocations to collect timing and memory statistics and to -# capture the results in a consistent set of output files for display -# and analysis. -# - -# -# __COPYRIGHT__ -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - -import getopt -import glob -import os -import re -import shutil -import sys -import tempfile -import time -import subprocess - -def HACK_for_exec(cmd, *args): - """ - For some reason, Python won't allow an exec() within a function - that also declares an internal function (including lambda functions). - This function is a hack that calls exec() in a function with no - internal functions. - """ - if not args: exec(cmd) - elif len(args) == 1: exec(cmd, args[0]) - else: exec(cmd, args[0], args[1]) - -class Plotter(object): - def increment_size(self, largest): - """ - Return the size of each horizontal increment line for a specified - maximum value. This returns a value that will provide somewhere - between 5 and 9 horizontal lines on the graph, on some set of - boundaries that are multiples of 10/100/1000/etc. - """ - i = largest // 5 - if not i: - return largest - multiplier = 1 - while i >= 10: - i = i // 10 - multiplier = multiplier * 10 - return i * multiplier - - def max_graph_value(self, largest): - # Round up to next integer. - largest = int(largest) + 1 - increment = self.increment_size(largest) - return ((largest + increment - 1) // increment) * increment - -class Line(object): - def __init__(self, points, type, title, label, comment, fmt="%s %s"): - self.points = points - self.type = type - self.title = title - self.label = label - self.comment = comment - self.fmt = fmt - - def print_label(self, inx, x, y): - if self.label: - print('set label %s "%s" at %0.1f,%0.1f right' % (inx, self.label, x, y)) - - def plot_string(self): - if self.title: - title_string = 'title "%s"' % self.title - else: - title_string = 'notitle' - return "'-' %s with lines lt %s" % (title_string, self.type) - - def print_points(self, fmt=None): - if fmt is None: - fmt = self.fmt - if self.comment: - print('# %s' % self.comment) - for x, y in self.points: - # If y is None, it usually represents some kind of break - # in the line's index number. We might want to represent - # this some way rather than just drawing the line straight - # between the two points on either side. - if y is not None: - print(fmt % (x, y)) - print('e') - - def get_x_values(self): - return [ p[0] for p in self.points ] - - def get_y_values(self): - return [ p[1] for p in self.points ] - -class Gnuplotter(Plotter): - - def __init__(self, title, key_location): - self.lines = [] - self.title = title - self.key_location = key_location - - def line(self, points, type, title=None, label=None, comment=None, fmt='%s %s'): - if points: - line = Line(points, type, title, label, comment, fmt) - self.lines.append(line) - - def plot_string(self, line): - return line.plot_string() - - def vertical_bar(self, x, type, label, comment): - if self.get_min_x() <= x <= self.get_max_x(): - points = [(x, 0), (x, self.max_graph_value(self.get_max_y()))] - self.line(points, type, label, comment) - - def get_all_x_values(self): - result = [] - for line in self.lines: - result.extend(line.get_x_values()) - return [r for r in result if r is not None] - - def get_all_y_values(self): - result = [] - for line in self.lines: - result.extend(line.get_y_values()) - return [r for r in result if r is not None] - - def get_min_x(self): - try: - return self.min_x - except AttributeError: - try: - self.min_x = min(self.get_all_x_values()) - except ValueError: - self.min_x = 0 - return self.min_x - - def get_max_x(self): - try: - return self.max_x - except AttributeError: - try: - self.max_x = max(self.get_all_x_values()) - except ValueError: - self.max_x = 0 - return self.max_x - - def get_min_y(self): - try: - return self.min_y - except AttributeError: - try: - self.min_y = min(self.get_all_y_values()) - except ValueError: - self.min_y = 0 - return self.min_y - - def get_max_y(self): - try: - return self.max_y - except AttributeError: - try: - self.max_y = max(self.get_all_y_values()) - except ValueError: - self.max_y = 0 - return self.max_y - - def draw(self): - - if not self.lines: - return - - if self.title: - print('set title "%s"' % self.title) - print('set key %s' % self.key_location) - - min_y = self.get_min_y() - max_y = self.max_graph_value(self.get_max_y()) - incr = (max_y - min_y) / 10.0 - start = min_y + (max_y / 2.0) + (2.0 * incr) - position = [ start - (i * incr) for i in range(5) ] - - inx = 1 - for line in self.lines: - line.print_label(inx, line.points[0][0]-1, - position[(inx-1) % len(position)]) - inx += 1 - - plot_strings = [ self.plot_string(l) for l in self.lines ] - print('plot ' + ', \\\n '.join(plot_strings)) - - for line in self.lines: - line.print_points() - - - -def untar(fname): - import tarfile - tar = tarfile.open(name=fname, mode='r') - for tarinfo in tar: - tar.extract(tarinfo) - tar.close() - -def unzip(fname): - import zipfile - zf = zipfile.ZipFile(fname, 'r') - for name in zf.namelist(): - dir = os.path.dirname(name) - try: - os.makedirs(dir) - except: - pass - with open(name, 'wb') as f: - f.write(zf.read(name)) - -def read_tree(dir): - for dirpath, dirnames, filenames in os.walk(dir): - for fn in filenames: - fn = os.path.join(dirpath, fn) - if os.path.isfile(fn): - with open(fn, 'rb') as f: - f.read() - -def redirect_to_file(command, log): - return '%s > %s 2>&1' % (command, log) - -def tee_to_file(command, log): - return '%s 2>&1 | tee %s' % (command, log) - - - -class SConsTimer(object): - """ - Usage: scons-time SUBCOMMAND [ARGUMENTS] - Type "scons-time help SUBCOMMAND" for help on a specific subcommand. - - Available subcommands: - func Extract test-run data for a function - help Provides help - mem Extract --debug=memory data from test runs - obj Extract --debug=count data from test runs - time Extract --debug=time data from test runs - run Runs a test configuration - """ - - name = 'scons-time' - name_spaces = ' '*len(name) - - def makedict(**kw): - return kw - - default_settings = makedict( - chdir = None, - config_file = None, - initial_commands = [], - key_location = 'bottom left', - orig_cwd = os.getcwd(), - outdir = None, - prefix = '', - python = '"%s"' % sys.executable, - redirect = redirect_to_file, - scons = None, - scons_flags = '--debug=count --debug=memory --debug=time --debug=memoizer', - scons_lib_dir = None, - scons_wrapper = None, - startup_targets = '--help', - subdir = None, - subversion_url = None, - svn = 'svn', - svn_co_flag = '-q', - tar = 'tar', - targets = '', - targets0 = None, - targets1 = None, - targets2 = None, - title = None, - unzip = 'unzip', - verbose = False, - vertical_bars = [], - - unpack_map = { - '.tar.gz' : (untar, '%(tar)s xzf %%s'), - '.tgz' : (untar, '%(tar)s xzf %%s'), - '.tar' : (untar, '%(tar)s xf %%s'), - '.zip' : (unzip, '%(unzip)s %%s'), - }, - ) - - run_titles = [ - 'Startup', - 'Full build', - 'Up-to-date build', - ] - - run_commands = [ - '%(python)s %(scons_wrapper)s %(scons_flags)s --profile=%(prof0)s %(targets0)s', - '%(python)s %(scons_wrapper)s %(scons_flags)s --profile=%(prof1)s %(targets1)s', - '%(python)s %(scons_wrapper)s %(scons_flags)s --profile=%(prof2)s %(targets2)s', - ] - - stages = [ - 'pre-read', - 'post-read', - 'pre-build', - 'post-build', - ] - - stage_strings = { - 'pre-read' : 'Memory before reading SConscript files:', - 'post-read' : 'Memory after reading SConscript files:', - 'pre-build' : 'Memory before building targets:', - 'post-build' : 'Memory after building targets:', - } - - memory_string_all = 'Memory ' - - default_stage = stages[-1] - - time_strings = { - 'total' : 'Total build time', - 'SConscripts' : 'Total SConscript file execution time', - 'SCons' : 'Total SCons execution time', - 'commands' : 'Total command execution time', - } - - time_string_all = 'Total .* time' - - # - - def __init__(self): - self.__dict__.update(self.default_settings) - - # Functions for displaying and executing commands. - - def subst(self, x, dictionary): - try: - return x % dictionary - except TypeError: - # x isn't a string (it's probably a Python function), - # so just return it. - return x - - def subst_variables(self, command, dictionary): - """ - Substitutes (via the format operator) the values in the specified - dictionary into the specified command. - - The command can be an (action, string) tuple. In all cases, we - perform substitution on strings and don't worry if something isn't - a string. (It's probably a Python function to be executed.) - """ - try: - command + '' - except TypeError: - action = command[0] - string = command[1] - args = command[2:] - else: - action = command - string = action - args = (()) - action = self.subst(action, dictionary) - string = self.subst(string, dictionary) - return (action, string, args) - - def _do_not_display(self, msg, *args): - pass - - def display(self, msg, *args): - """ - Displays the specified message. - - Each message is prepended with a standard prefix of our name - plus the time. - """ - if callable(msg): - msg = msg(*args) - else: - msg = msg % args - if msg is None: - return - fmt = '%s[%s]: %s\n' - sys.stdout.write(fmt % (self.name, time.strftime('%H:%M:%S'), msg)) - - def _do_not_execute(self, action, *args): - pass - - def execute(self, action, *args): - """ - Executes the specified action. - - The action is called if it's a callable Python function, and - otherwise passed to os.system(). - """ - if callable(action): - action(*args) - else: - os.system(action % args) - - def run_command_list(self, commands, dict): - """ - Executes a list of commands, substituting values from the - specified dictionary. - """ - commands = [ self.subst_variables(c, dict) for c in commands ] - for action, string, args in commands: - self.display(string, *args) - sys.stdout.flush() - status = self.execute(action, *args) - if status: - sys.exit(status) - - def log_display(self, command, log): - command = self.subst(command, self.__dict__) - if log: - command = self.redirect(command, log) - return command - - def log_execute(self, command, log): - command = self.subst(command, self.__dict__) - p = os.popen(command) - output = p.read() - p.close() - #TODO: convert to subrocess, os.popen is obsolete. This didn't work: - #process = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True) - #output = process.stdout.read() - #process.stdout.close() - #process.wait() - if self.verbose: - sys.stdout.write(output) - # TODO: Figure out - # Not sure we need to write binary here - with open(log, 'w') as f: - f.write(str(output)) - - def archive_splitext(self, path): - """ - Splits an archive name into a filename base and extension. - - This is like os.path.splitext() (which it calls) except that it - also looks for '.tar.gz' and treats it as an atomic extensions. - """ - if path.endswith('.tar.gz'): - return path[:-7], path[-7:] - else: - return os.path.splitext(path) - - def args_to_files(self, args, tail=None): - """ - Takes a list of arguments, expands any glob patterns, and - returns the last "tail" files from the list. - """ - files = [] - for a in args: - files.extend(sorted(glob.glob(a))) - - if tail: - files = files[-tail:] - - return files - - def ascii_table(self, files, columns, - line_function, file_function=lambda x: x, - *args, **kw): - - header_fmt = ' '.join(['%12s'] * len(columns)) - line_fmt = header_fmt + ' %s' - - print(header_fmt % columns) - - for file in files: - t = line_function(file, *args, **kw) - if t is None: - t = [] - diff = len(columns) - len(t) - if diff > 0: - t += [''] * diff - t.append(file_function(file)) - print(line_fmt % tuple(t)) - - def collect_results(self, files, function, *args, **kw): - results = {} - - for file in files: - base = os.path.splitext(file)[0] - run, index = base.split('-')[-2:] - - run = int(run) - index = int(index) - - value = function(file, *args, **kw) - - try: - r = results[index] - except KeyError: - r = [] - results[index] = r - r.append((run, value)) - - return results - - def doc_to_help(self, obj): - """ - Translates an object's __doc__ string into help text. - - This strips a consistent number of spaces from each line in the - help text, essentially "outdenting" the text to the left-most - column. - """ - doc = obj.__doc__ - if doc is None: - return '' - return self.outdent(doc) - - def find_next_run_number(self, dir, prefix): - """ - Returns the next run number in a directory for the specified prefix. - - Examines the contents the specified directory for files with the - specified prefix, extracts the run numbers from each file name, - and returns the next run number after the largest it finds. - """ - x = re.compile(re.escape(prefix) + '-([0-9]+).*') - matches = [x.match(e) for e in os.listdir(dir)] - matches = [_f for _f in matches if _f] - if not matches: - return 0 - run_numbers = [int(m.group(1)) for m in matches] - return int(max(run_numbers)) + 1 - - def gnuplot_results(self, results, fmt='%s %.3f'): - """ - Prints out a set of results in Gnuplot format. - """ - gp = Gnuplotter(self.title, self.key_location) - - for i in sorted(results.keys()): - try: - t = self.run_titles[i] - except IndexError: - t = '??? %s ???' % i - results[i].sort() - gp.line(results[i], i+1, t, None, t, fmt=fmt) - - for bar_tuple in self.vertical_bars: - try: - x, type, label, comment = bar_tuple - except ValueError: - x, type, label = bar_tuple - comment = label - gp.vertical_bar(x, type, label, comment) - - gp.draw() - - def logfile_name(self, invocation): - """ - Returns the absolute path of a log file for the specificed - invocation number. - """ - name = self.prefix_run + '-%d.log' % invocation - return os.path.join(self.outdir, name) - - def outdent(self, s): - """ - Strip as many spaces from each line as are found at the beginning - of the first line in the list. - """ - lines = s.split('\n') - if lines[0] == '': - lines = lines[1:] - spaces = re.match(' *', lines[0]).group(0) - def strip_initial_spaces(l, s=spaces): - if l.startswith(spaces): - l = l[len(spaces):] - return l - return '\n'.join([ strip_initial_spaces(l) for l in lines ]) + '\n' - - def profile_name(self, invocation): - """ - Returns the absolute path of a profile file for the specified - invocation number. - """ - name = self.prefix_run + '-%d.prof' % invocation - return os.path.join(self.outdir, name) - - def set_env(self, key, value): - os.environ[key] = value - - # - - def get_debug_times(self, file, time_string=None): - """ - Fetch times from the --debug=time strings in the specified file. - """ - if time_string is None: - search_string = self.time_string_all - else: - search_string = time_string - with open(file) as f: - contents = f.read() - if not contents: - sys.stderr.write('file %s has no contents!\n' % repr(file)) - return None - result = re.findall(r'%s: ([\d.]*)' % search_string, contents)[-4:] - result = [ float(r) for r in result ] - if time_string is not None: - try: - result = result[0] - except IndexError: - sys.stderr.write('file %s has no results!\n' % repr(file)) - return None - return result - - def get_function_profile(self, file, function): - """ - Returns the file, line number, function name, and cumulative time. - """ - try: - import pstats - except ImportError as e: - sys.stderr.write('%s: func: %s\n' % (self.name, e)) - sys.stderr.write('%s This version of Python is missing the profiler.\n' % self.name_spaces) - sys.stderr.write('%s Cannot use the "func" subcommand.\n' % self.name_spaces) - sys.exit(1) - statistics = pstats.Stats(file).stats - matches = [ e for e in statistics.items() if e[0][2] == function ] - r = matches[0] - return r[0][0], r[0][1], r[0][2], r[1][3] - - def get_function_time(self, file, function): - """ - Returns just the cumulative time for the specified function. - """ - return self.get_function_profile(file, function)[3] - - def get_memory(self, file, memory_string=None): - """ - Returns a list of integers of the amount of memory used. The - default behavior is to return all the stages. - """ - if memory_string is None: - search_string = self.memory_string_all - else: - search_string = memory_string - with open(file) as f: - lines = f.readlines() - lines = [ l for l in lines if l.startswith(search_string) ][-4:] - result = [ int(l.split()[-1]) for l in lines[-4:] ] - if len(result) == 1: - result = result[0] - return result - - def get_object_counts(self, file, object_name, index=None): - """ - Returns the counts of the specified object_name. - """ - object_string = ' ' + object_name + '\n' - with open(file) as f: - lines = f.readlines() - line = [ l for l in lines if l.endswith(object_string) ][0] - result = [ int(field) for field in line.split()[:4] ] - if index is not None: - result = result[index] - return result - - - command_alias = {} - - def execute_subcommand(self, argv): - """ - Executes the do_*() function for the specified subcommand (argv[0]). - """ - if not argv: - return - cmdName = self.command_alias.get(argv[0], argv[0]) - try: - func = getattr(self, 'do_' + cmdName) - except AttributeError: - return self.default(argv) - try: - return func(argv) - except TypeError as e: - sys.stderr.write("%s %s: %s\n" % (self.name, cmdName, e)) - import traceback - traceback.print_exc(file=sys.stderr) - sys.stderr.write("Try '%s help %s'\n" % (self.name, cmdName)) - - def default(self, argv): - """ - The default behavior for an unknown subcommand. Prints an - error message and exits. - """ - sys.stderr.write('%s: Unknown subcommand "%s".\n' % (self.name, argv[0])) - sys.stderr.write('Type "%s help" for usage.\n' % self.name) - sys.exit(1) - - # - - def do_help(self, argv): - """ - """ - if argv[1:]: - for arg in argv[1:]: - try: - func = getattr(self, 'do_' + arg) - except AttributeError: - sys.stderr.write('%s: No help for "%s"\n' % (self.name, arg)) - else: - try: - help = getattr(self, 'help_' + arg) - except AttributeError: - sys.stdout.write(self.doc_to_help(func)) - sys.stdout.flush() - else: - help() - else: - doc = self.doc_to_help(self.__class__) - if doc: - sys.stdout.write(doc) - sys.stdout.flush() - return None - - # - - def help_func(self): - help = """\ - Usage: scons-time func [OPTIONS] FILE [...] - - -C DIR, --chdir=DIR Change to DIR before looking for files - -f FILE, --file=FILE Read configuration from specified FILE - --fmt=FORMAT, --format=FORMAT Print data in specified FORMAT - --func=NAME, --function=NAME Report time for function NAME - -h, --help Print this help and exit - -p STRING, --prefix=STRING Use STRING as log file/profile prefix - -t NUMBER, --tail=NUMBER Only report the last NUMBER files - --title=TITLE Specify the output plot TITLE - """ - sys.stdout.write(self.outdent(help)) - sys.stdout.flush() - - def do_func(self, argv): - """ - """ - format = 'ascii' - function_name = '_main' - tail = None - - short_opts = '?C:f:hp:t:' - - long_opts = [ - 'chdir=', - 'file=', - 'fmt=', - 'format=', - 'func=', - 'function=', - 'help', - 'prefix=', - 'tail=', - 'title=', - ] - - opts, args = getopt.getopt(argv[1:], short_opts, long_opts) - - for o, a in opts: - if o in ('-C', '--chdir'): - self.chdir = a - elif o in ('-f', '--file'): - self.config_file = a - elif o in ('--fmt', '--format'): - format = a - elif o in ('--func', '--function'): - function_name = a - elif o in ('-?', '-h', '--help'): - self.do_help(['help', 'func']) - sys.exit(0) - elif o in ('--max',): - max_time = int(a) - elif o in ('-p', '--prefix'): - self.prefix = a - elif o in ('-t', '--tail'): - tail = int(a) - elif o in ('--title',): - self.title = a - - if self.config_file: - with open(self.config_file, 'r') as f: - config = f.read() - exec(config, self.__dict__) - - if self.chdir: - os.chdir(self.chdir) - - if not args: - - pattern = '%s*.prof' % self.prefix - args = self.args_to_files([pattern], tail) - - if not args: - if self.chdir: - directory = self.chdir - else: - directory = os.getcwd() - - sys.stderr.write('%s: func: No arguments specified.\n' % self.name) - sys.stderr.write('%s No %s*.prof files found in "%s".\n' % (self.name_spaces, self.prefix, directory)) - sys.stderr.write('%s Type "%s help func" for help.\n' % (self.name_spaces, self.name)) - sys.exit(1) - - else: - - args = self.args_to_files(args, tail) - - cwd_ = os.getcwd() + os.sep - - if format == 'ascii': - - for file in args: - try: - f, line, func, time = \ - self.get_function_profile(file, function_name) - except ValueError as e: - sys.stderr.write("%s: func: %s: %s\n" % - (self.name, file, e)) - else: - if f.startswith(cwd_): - f = f[len(cwd_):] - print("%.3f %s:%d(%s)" % (time, f, line, func)) - - elif format == 'gnuplot': - - results = self.collect_results(args, self.get_function_time, - function_name) - - self.gnuplot_results(results) - - else: - - sys.stderr.write('%s: func: Unknown format "%s".\n' % (self.name, format)) - sys.exit(1) - - # - - def help_mem(self): - help = """\ - Usage: scons-time mem [OPTIONS] FILE [...] - - -C DIR, --chdir=DIR Change to DIR before looking for files - -f FILE, --file=FILE Read configuration from specified FILE - --fmt=FORMAT, --format=FORMAT Print data in specified FORMAT - -h, --help Print this help and exit - -p STRING, --prefix=STRING Use STRING as log file/profile prefix - --stage=STAGE Plot memory at the specified stage: - pre-read, post-read, pre-build, - post-build (default: post-build) - -t NUMBER, --tail=NUMBER Only report the last NUMBER files - --title=TITLE Specify the output plot TITLE - """ - sys.stdout.write(self.outdent(help)) - sys.stdout.flush() - - def do_mem(self, argv): - - format = 'ascii' - logfile_path = lambda x: x - stage = self.default_stage - tail = None - - short_opts = '?C:f:hp:t:' - - long_opts = [ - 'chdir=', - 'file=', - 'fmt=', - 'format=', - 'help', - 'prefix=', - 'stage=', - 'tail=', - 'title=', - ] - - opts, args = getopt.getopt(argv[1:], short_opts, long_opts) - - for o, a in opts: - if o in ('-C', '--chdir'): - self.chdir = a - elif o in ('-f', '--file'): - self.config_file = a - elif o in ('--fmt', '--format'): - format = a - elif o in ('-?', '-h', '--help'): - self.do_help(['help', 'mem']) - sys.exit(0) - elif o in ('-p', '--prefix'): - self.prefix = a - elif o in ('--stage',): - if a not in self.stages: - sys.stderr.write('%s: mem: Unrecognized stage "%s".\n' % (self.name, a)) - sys.exit(1) - stage = a - elif o in ('-t', '--tail'): - tail = int(a) - elif o in ('--title',): - self.title = a - - if self.config_file: - with open(self.config_file, 'r') as f: - config = f.read() - HACK_for_exec(config, self.__dict__) - - if self.chdir: - os.chdir(self.chdir) - logfile_path = lambda x: os.path.join(self.chdir, x) - - if not args: - - pattern = '%s*.log' % self.prefix - args = self.args_to_files([pattern], tail) - - if not args: - if self.chdir: - directory = self.chdir - else: - directory = os.getcwd() - - sys.stderr.write('%s: mem: No arguments specified.\n' % self.name) - sys.stderr.write('%s No %s*.log files found in "%s".\n' % (self.name_spaces, self.prefix, directory)) - sys.stderr.write('%s Type "%s help mem" for help.\n' % (self.name_spaces, self.name)) - sys.exit(1) - - else: - - args = self.args_to_files(args, tail) - - cwd_ = os.getcwd() + os.sep - - if format == 'ascii': - - self.ascii_table(args, tuple(self.stages), self.get_memory, logfile_path) - - elif format == 'gnuplot': - - results = self.collect_results(args, self.get_memory, - self.stage_strings[stage]) - - self.gnuplot_results(results) - - else: - - sys.stderr.write('%s: mem: Unknown format "%s".\n' % (self.name, format)) - sys.exit(1) - - return 0 - - # - - def help_obj(self): - help = """\ - Usage: scons-time obj [OPTIONS] OBJECT FILE [...] - - -C DIR, --chdir=DIR Change to DIR before looking for files - -f FILE, --file=FILE Read configuration from specified FILE - --fmt=FORMAT, --format=FORMAT Print data in specified FORMAT - -h, --help Print this help and exit - -p STRING, --prefix=STRING Use STRING as log file/profile prefix - --stage=STAGE Plot memory at the specified stage: - pre-read, post-read, pre-build, - post-build (default: post-build) - -t NUMBER, --tail=NUMBER Only report the last NUMBER files - --title=TITLE Specify the output plot TITLE - """ - sys.stdout.write(self.outdent(help)) - sys.stdout.flush() - - def do_obj(self, argv): - - format = 'ascii' - logfile_path = lambda x: x - stage = self.default_stage - tail = None - - short_opts = '?C:f:hp:t:' - - long_opts = [ - 'chdir=', - 'file=', - 'fmt=', - 'format=', - 'help', - 'prefix=', - 'stage=', - 'tail=', - 'title=', - ] - - opts, args = getopt.getopt(argv[1:], short_opts, long_opts) - - for o, a in opts: - if o in ('-C', '--chdir'): - self.chdir = a - elif o in ('-f', '--file'): - self.config_file = a - elif o in ('--fmt', '--format'): - format = a - elif o in ('-?', '-h', '--help'): - self.do_help(['help', 'obj']) - sys.exit(0) - elif o in ('-p', '--prefix'): - self.prefix = a - elif o in ('--stage',): - if a not in self.stages: - sys.stderr.write('%s: obj: Unrecognized stage "%s".\n' % (self.name, a)) - sys.stderr.write('%s Type "%s help obj" for help.\n' % (self.name_spaces, self.name)) - sys.exit(1) - stage = a - elif o in ('-t', '--tail'): - tail = int(a) - elif o in ('--title',): - self.title = a - - if not args: - sys.stderr.write('%s: obj: Must specify an object name.\n' % self.name) - sys.stderr.write('%s Type "%s help obj" for help.\n' % (self.name_spaces, self.name)) - sys.exit(1) - - object_name = args.pop(0) - - if self.config_file: - with open(self.config_file, 'r') as f: - config = f.read() - HACK_for_exec(config, self.__dict__) - - if self.chdir: - os.chdir(self.chdir) - logfile_path = lambda x: os.path.join(self.chdir, x) - - if not args: - - pattern = '%s*.log' % self.prefix - args = self.args_to_files([pattern], tail) - - if not args: - if self.chdir: - directory = self.chdir - else: - directory = os.getcwd() - - sys.stderr.write('%s: obj: No arguments specified.\n' % self.name) - sys.stderr.write('%s No %s*.log files found in "%s".\n' % (self.name_spaces, self.prefix, directory)) - sys.stderr.write('%s Type "%s help obj" for help.\n' % (self.name_spaces, self.name)) - sys.exit(1) - - else: - - args = self.args_to_files(args, tail) - - cwd_ = os.getcwd() + os.sep - - if format == 'ascii': - - self.ascii_table(args, tuple(self.stages), self.get_object_counts, logfile_path, object_name) - - elif format == 'gnuplot': - - stage_index = 0 - for s in self.stages: - if stage == s: - break - stage_index = stage_index + 1 - - results = self.collect_results(args, self.get_object_counts, - object_name, stage_index) - - self.gnuplot_results(results) - - else: - - sys.stderr.write('%s: obj: Unknown format "%s".\n' % (self.name, format)) - sys.exit(1) - - return 0 - - # - - def help_run(self): - help = """\ - Usage: scons-time run [OPTIONS] [FILE ...] - - --chdir=DIR Name of unpacked directory for chdir - -f FILE, --file=FILE Read configuration from specified FILE - -h, --help Print this help and exit - -n, --no-exec No execute, just print command lines - --number=NUMBER Put output in files for run NUMBER - --outdir=OUTDIR Put output files in OUTDIR - -p STRING, --prefix=STRING Use STRING as log file/profile prefix - --python=PYTHON Time using the specified PYTHON - -q, --quiet Don't print command lines - --scons=SCONS Time using the specified SCONS - --svn=URL, --subversion=URL Use SCons from Subversion URL - -v, --verbose Display output of commands - """ - sys.stdout.write(self.outdent(help)) - sys.stdout.flush() - - def do_run(self, argv): - """ - """ - run_number_list = [None] - - short_opts = '?f:hnp:qs:v' - - long_opts = [ - 'file=', - 'help', - 'no-exec', - 'number=', - 'outdir=', - 'prefix=', - 'python=', - 'quiet', - 'scons=', - 'svn=', - 'subdir=', - 'subversion=', - 'verbose', - ] - - opts, args = getopt.getopt(argv[1:], short_opts, long_opts) - - for o, a in opts: - if o in ('-f', '--file'): - self.config_file = a - elif o in ('-?', '-h', '--help'): - self.do_help(['help', 'run']) - sys.exit(0) - elif o in ('-n', '--no-exec'): - self.execute = self._do_not_execute - elif o in ('--number',): - run_number_list = self.split_run_numbers(a) - elif o in ('--outdir',): - self.outdir = a - elif o in ('-p', '--prefix'): - self.prefix = a - elif o in ('--python',): - self.python = a - elif o in ('-q', '--quiet'): - self.display = self._do_not_display - elif o in ('-s', '--subdir'): - self.subdir = a - elif o in ('--scons',): - self.scons = a - elif o in ('--svn', '--subversion'): - self.subversion_url = a - elif o in ('-v', '--verbose'): - self.redirect = tee_to_file - self.verbose = True - self.svn_co_flag = '' - - if not args and not self.config_file: - sys.stderr.write('%s: run: No arguments or -f config file specified.\n' % self.name) - sys.stderr.write('%s Type "%s help run" for help.\n' % (self.name_spaces, self.name)) - sys.exit(1) - - if self.config_file: - with open(self.config_file, 'r') as f: - config = f.read() - exec(config, self.__dict__) - - if args: - self.archive_list = args - - archive_file_name = os.path.split(self.archive_list[0])[1] - - if not self.subdir: - self.subdir = self.archive_splitext(archive_file_name)[0] - - if not self.prefix: - self.prefix = self.archive_splitext(archive_file_name)[0] - - prepare = None - if self.subversion_url: - prepare = self.prep_subversion_run - - for run_number in run_number_list: - self.individual_run(run_number, self.archive_list, prepare) - - def split_run_numbers(self, s): - result = [] - for n in s.split(','): - try: - x, y = n.split('-') - except ValueError: - result.append(int(n)) - else: - result.extend(list(range(int(x), int(y)+1))) - return result - - def scons_path(self, dir): - return os.path.join(dir, 'src', 'script', 'scons.py') - - def scons_lib_dir_path(self, dir): - return os.path.join(dir, 'src', 'engine') - - def prep_subversion_run(self, commands, removals): - self.svn_tmpdir = tempfile.mkdtemp(prefix=self.name + '-svn-') - removals.append((shutil.rmtree, 'rm -rf %%s', self.svn_tmpdir)) - - self.scons = self.scons_path(self.svn_tmpdir) - self.scons_lib_dir = self.scons_lib_dir_path(self.svn_tmpdir) - - commands.extend([ - '%(svn)s co %(svn_co_flag)s -r %(run_number)s %(subversion_url)s %(svn_tmpdir)s', - ]) - - def individual_run(self, run_number, archive_list, prepare=None): - """ - Performs an individual run of the default SCons invocations. - """ - - commands = [] - removals = [] - - if prepare: - prepare(commands, removals) - - save_scons = self.scons - save_scons_wrapper = self.scons_wrapper - save_scons_lib_dir = self.scons_lib_dir - - if self.outdir is None: - self.outdir = self.orig_cwd - elif not os.path.isabs(self.outdir): - self.outdir = os.path.join(self.orig_cwd, self.outdir) - - if self.scons is None: - self.scons = self.scons_path(self.orig_cwd) - - if self.scons_lib_dir is None: - self.scons_lib_dir = self.scons_lib_dir_path(self.orig_cwd) - - if self.scons_wrapper is None: - self.scons_wrapper = self.scons - - if not run_number: - run_number = self.find_next_run_number(self.outdir, self.prefix) - - self.run_number = str(run_number) - - self.prefix_run = self.prefix + '-%03d' % run_number - - if self.targets0 is None: - self.targets0 = self.startup_targets - if self.targets1 is None: - self.targets1 = self.targets - if self.targets2 is None: - self.targets2 = self.targets - - self.tmpdir = tempfile.mkdtemp(prefix=self.name + '-') - - commands.extend([ - (os.chdir, 'cd %%s', self.tmpdir), - ]) - - for archive in archive_list: - if not os.path.isabs(archive): - archive = os.path.join(self.orig_cwd, archive) - if os.path.isdir(archive): - dest = os.path.split(archive)[1] - commands.append((shutil.copytree, 'cp -r %%s %%s', archive, dest)) - else: - suffix = self.archive_splitext(archive)[1] - unpack_command = self.unpack_map.get(suffix) - if not unpack_command: - dest = os.path.split(archive)[1] - commands.append((shutil.copyfile, 'cp %%s %%s', archive, dest)) - else: - commands.append(unpack_command + (archive,)) - - commands.extend([ - (os.chdir, 'cd %%s', self.subdir), - ]) - - commands.extend(self.initial_commands) - - commands.extend([ - (lambda: read_tree('.'), - 'find * -type f | xargs cat > /dev/null'), - - (self.set_env, 'export %%s=%%s', - 'SCONS_LIB_DIR', self.scons_lib_dir), - - '%(python)s %(scons_wrapper)s --version', - ]) - - index = 0 - for run_command in self.run_commands: - setattr(self, 'prof%d' % index, self.profile_name(index)) - c = ( - self.log_execute, - self.log_display, - run_command, - self.logfile_name(index), - ) - commands.append(c) - index = index + 1 - - commands.extend([ - (os.chdir, 'cd %%s', self.orig_cwd), - ]) - - if not os.environ.get('PRESERVE'): - commands.extend(removals) - commands.append((shutil.rmtree, 'rm -rf %%s', self.tmpdir)) - - self.run_command_list(commands, self.__dict__) - - self.scons = save_scons - self.scons_lib_dir = save_scons_lib_dir - self.scons_wrapper = save_scons_wrapper - - # - - def help_time(self): - help = """\ - Usage: scons-time time [OPTIONS] FILE [...] - - -C DIR, --chdir=DIR Change to DIR before looking for files - -f FILE, --file=FILE Read configuration from specified FILE - --fmt=FORMAT, --format=FORMAT Print data in specified FORMAT - -h, --help Print this help and exit - -p STRING, --prefix=STRING Use STRING as log file/profile prefix - -t NUMBER, --tail=NUMBER Only report the last NUMBER files - --which=TIMER Plot timings for TIMER: total, - SConscripts, SCons, commands. - """ - sys.stdout.write(self.outdent(help)) - sys.stdout.flush() - - def do_time(self, argv): - - format = 'ascii' - logfile_path = lambda x: x - tail = None - which = 'total' - - short_opts = '?C:f:hp:t:' - - long_opts = [ - 'chdir=', - 'file=', - 'fmt=', - 'format=', - 'help', - 'prefix=', - 'tail=', - 'title=', - 'which=', - ] - - opts, args = getopt.getopt(argv[1:], short_opts, long_opts) - - for o, a in opts: - if o in ('-C', '--chdir'): - self.chdir = a - elif o in ('-f', '--file'): - self.config_file = a - elif o in ('--fmt', '--format'): - format = a - elif o in ('-?', '-h', '--help'): - self.do_help(['help', 'time']) - sys.exit(0) - elif o in ('-p', '--prefix'): - self.prefix = a - elif o in ('-t', '--tail'): - tail = int(a) - elif o in ('--title',): - self.title = a - elif o in ('--which',): - if a not in list(self.time_strings.keys()): - sys.stderr.write('%s: time: Unrecognized timer "%s".\n' % (self.name, a)) - sys.stderr.write('%s Type "%s help time" for help.\n' % (self.name_spaces, self.name)) - sys.exit(1) - which = a - - if self.config_file: - with open(self.config_file, 'r') as f: - config = f.read() - HACK_for_exec(config, self.__dict__) - - if self.chdir: - os.chdir(self.chdir) - logfile_path = lambda x: os.path.join(self.chdir, x) - - if not args: - - pattern = '%s*.log' % self.prefix - args = self.args_to_files([pattern], tail) - - if not args: - if self.chdir: - directory = self.chdir - else: - directory = os.getcwd() - - sys.stderr.write('%s: time: No arguments specified.\n' % self.name) - sys.stderr.write('%s No %s*.log files found in "%s".\n' % (self.name_spaces, self.prefix, directory)) - sys.stderr.write('%s Type "%s help time" for help.\n' % (self.name_spaces, self.name)) - sys.exit(1) - - else: - - args = self.args_to_files(args, tail) - - cwd_ = os.getcwd() + os.sep - - if format == 'ascii': - - columns = ("Total", "SConscripts", "SCons", "commands") - self.ascii_table(args, columns, self.get_debug_times, logfile_path) - - elif format == 'gnuplot': - - results = self.collect_results(args, self.get_debug_times, - self.time_strings[which]) - - self.gnuplot_results(results, fmt='%s %.6f') - - else: - - sys.stderr.write('%s: time: Unknown format "%s".\n' % (self.name, format)) - sys.exit(1) - -if __name__ == '__main__': - opts, args = getopt.getopt(sys.argv[1:], 'h?V', ['help', 'version']) - - ST = SConsTimer() - - for o, a in opts: - if o in ('-?', '-h', '--help'): - ST.do_help(['help']) - sys.exit(0) - elif o in ('-V', '--version'): - sys.stdout.write('scons-time version\n') - sys.exit(0) - - if not args: - sys.stderr.write('Type "%s help" for usage.\n' % ST.name) - sys.exit(1) - - ST.execute_subcommand(args) - -# Local Variables: -# tab-width:4 -# indent-tabs-mode:nil -# End: -# vim: set expandtab tabstop=4 shiftwidth=4: -- cgit v0.12 From 3c542bcc3f67fe1daaa7a822f856afe7086dc974 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 21 Mar 2020 17:36:23 -0700 Subject: move scons-configure-cache logic into SCons.Utilities.ConfigureCache:main and update references and add entry_point in setup.cfg --- scripts/scons-configure-cache.py | 181 +++++++------------------- setup.cfg | 1 + src/engine/SCons/Utilities/ConfigureCache.py | 182 +++++++++++++++++++++++++++ 3 files changed, 230 insertions(+), 134 deletions(-) create mode 100644 src/engine/SCons/Utilities/ConfigureCache.py diff --git a/scripts/scons-configure-cache.py b/scripts/scons-configure-cache.py index 80783a9..62cab56 100644 --- a/scripts/scons-configure-cache.py +++ b/scripts/scons-configure-cache.py @@ -30,12 +30,6 @@ The files are split into directories named by the first few digits of the signature. The prefix length used for directory names can be changed by this script. """ - -import argparse -import glob -import json -import os - __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" __version__ = "__VERSION__" @@ -49,134 +43,53 @@ __date__ = "__DATE__" __developer__ = "__DEVELOPER__" -def rearrange_cache_entries(current_prefix_len, new_prefix_len): - """Move cache files if prefix length changed. - - Move the existing cache files to new directories of the - appropriate name length and clean up the old directories. - """ - print('Changing prefix length from', current_prefix_len, - 'to', new_prefix_len) - dirs = set() - old_dirs = set() - for file in glob.iglob(os.path.join('*', '*')): - name = os.path.basename(file) - dname = name[:current_prefix_len].upper() - if dname not in old_dirs: - print('Migrating', dname) - old_dirs.add(dname) - dname = name[:new_prefix_len].upper() - if dname not in dirs: - os.mkdir(dname) - dirs.add(dname) - os.rename(file, os.path.join(dname, name)) - - # Now delete the original directories - for dname in old_dirs: - os.rmdir(dname) - - -# The configuration dictionary should have one entry per entry in the -# cache config. The value of each entry should include the following: -# implicit - (optional) This is to allow adding a new config entry and also -# changing the behaviour of the system at the same time. This -# indicates the value the config entry would have had if it had -# been specified. -# default - The value the config entry should have if it wasn't previously -# specified -# command-line - parameters to pass to ArgumentParser.add_argument -# converter - (optional) Function to call if conversion is required -# if this configuration entry changes -config_entries = { - 'prefix_len': { - 'implicit': 1, - 'default': 2, - 'command-line': { - 'help': 'Length of cache file name used as subdirectory prefix', - 'metavar': '', - 'type': int - }, - 'converter': rearrange_cache_entries - } -} - - -def main(): - parser = argparse.ArgumentParser( - description='Modify the configuration of an scons cache directory', - epilog=''' - Unspecified options will not be changed unless they are not - set at all, in which case they are set to an appropriate default. - ''') - - parser.add_argument('cache-dir', help='Path to scons cache directory') - for param in config_entries: - parser.add_argument('--' + param.replace('_', '-'), - **config_entries[param]['command-line']) - parser.add_argument('--version', - action='version', - version='%(prog)s 1.0') - parser.add_argument('--show', - action="store_true", - help="show current configuration") - - # Get the command line as a dict without any of the unspecified entries. - args = dict([x for x in vars(parser.parse_args()).items() if x[1]]) - - # It seems somewhat strange to me, but positional arguments don't get the - - # in the name changed to _, whereas optional arguments do... - cache = args['cache-dir'] - if not os.path.isdir(cache): - raise RuntimeError("There is no cache directory named %s" % cache) - os.chdir(cache) - del args['cache-dir'] - - if not os.path.exists('config'): - # old config dirs did not have a 'config' file. Try to update. - # Validate the only files in the directory are directories 0-9, a-f - expected = ['{:X}'.format(x) for x in range(0, 16)] - if not set(os.listdir('.')).issubset(expected): - raise RuntimeError( - "%s does not look like a valid version 1 cache directory" % cache) - config = dict() - else: - with open('config') as conf: - config = json.load(conf) - - if args.get('show', None): - print("Current configuration in '%s':" % cache) - print(json.dumps(config, sort_keys=True, - indent=4, separators=(',', ': '))) - # in case of the show argument, emit some stats as well - file_count = 0 - for _, _, files in os.walk('.'): - file_count += len(files) - if file_count: # skip config file if it exists - file_count -= 1 - print("Cache contains %s files" % file_count) - del args['show'] - - # Find any keys that are not currently set but should be - for key in config_entries: - if key not in config: - if 'implicit' in config_entries[key]: - config[key] = config_entries[key]['implicit'] - else: - config[key] = config_entries[key]['default'] - if key not in args: - args[key] = config_entries[key]['default'] - - # Now go through each entry in args to see if it changes an existing config - # setting. - for key in args: - if args[key] != config[key]: - if 'converter' in config_entries[key]: - config_entries[key]['converter'](config[key], args[key]) - config[key] = args[key] - - # and write the updated config file - with open('config', 'w') as conf: - json.dump(config, conf) +import os +import sys + +# python compatibility check +if sys.version_info < (3, 5, 0): + msg = "scons: *** SCons version %s does not run under Python version %s.\n\ +Python >= 3.5 is required.\n" + sys.stderr.write(msg % (__version__, sys.version.split()[0])) + sys.exit(1) + +# Strip the script directory from sys.path so on case-insensitive +# (WIN32) systems Python doesn't think that the "scons" script is the +# "SCons" package. +script_dir = os.path.dirname(os.path.realpath(__file__)) +script_path = os.path.realpath(os.path.dirname(__file__)) +if script_path in sys.path: + sys.path.remove(script_path) + +libs = [] + +if "SCONS_LIB_DIR" in os.environ: + libs.append(os.environ["SCONS_LIB_DIR"]) + +# running from source takes 2nd priority (since 2.3.2), following SCONS_LIB_DIR +source_path = os.path.join(script_path, os.pardir, 'src', 'engine') +if os.path.isdir(source_path): + libs.append(source_path) + +# add local-install locations +local_version = 'scons-local-' + __version__ +local = 'scons-local' +if script_dir: + local_version = os.path.join(script_dir, local_version) + local = os.path.join(script_dir, local) +if os.path.isdir(local_version): + libs.append(os.path.abspath(local_version)) +if os.path.isdir(local): + libs.append(os.path.abspath(local)) + +scons_version = 'scons-%s' % __version__ + +sys.path = libs + sys.path + +############################################################################## +# END STANDARD SCons SCRIPT HEADER +############################################################################## +from SCons.Utilities.ConfigureCache import main if __name__ == "__main__": main() \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 8e55f10..2eb7728 100644 --- a/setup.cfg +++ b/setup.cfg @@ -54,6 +54,7 @@ packages = find: console_scripts = scons = SCons.Script.Main:main sconsign = SCons.Utilities.sconsign:main + scons-configure-cache = SCons.Utilities.ConfigureCache:main [options.package_data] diff --git a/src/engine/SCons/Utilities/ConfigureCache.py b/src/engine/SCons/Utilities/ConfigureCache.py new file mode 100644 index 0000000..80783a9 --- /dev/null +++ b/src/engine/SCons/Utilities/ConfigureCache.py @@ -0,0 +1,182 @@ +#! /usr/bin/env python +# +# SCons - a Software Constructor +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +"""Show or convert the configuration of an SCons cache directory. + +A cache of derived files is stored by file signature. +The files are split into directories named by the first few +digits of the signature. The prefix length used for directory +names can be changed by this script. +""" + +import argparse +import glob +import json +import os + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +__version__ = "__VERSION__" + +__build__ = "__BUILD__" + +__buildsys__ = "__BUILDSYS__" + +__date__ = "__DATE__" + +__developer__ = "__DEVELOPER__" + + +def rearrange_cache_entries(current_prefix_len, new_prefix_len): + """Move cache files if prefix length changed. + + Move the existing cache files to new directories of the + appropriate name length and clean up the old directories. + """ + print('Changing prefix length from', current_prefix_len, + 'to', new_prefix_len) + dirs = set() + old_dirs = set() + for file in glob.iglob(os.path.join('*', '*')): + name = os.path.basename(file) + dname = name[:current_prefix_len].upper() + if dname not in old_dirs: + print('Migrating', dname) + old_dirs.add(dname) + dname = name[:new_prefix_len].upper() + if dname not in dirs: + os.mkdir(dname) + dirs.add(dname) + os.rename(file, os.path.join(dname, name)) + + # Now delete the original directories + for dname in old_dirs: + os.rmdir(dname) + + +# The configuration dictionary should have one entry per entry in the +# cache config. The value of each entry should include the following: +# implicit - (optional) This is to allow adding a new config entry and also +# changing the behaviour of the system at the same time. This +# indicates the value the config entry would have had if it had +# been specified. +# default - The value the config entry should have if it wasn't previously +# specified +# command-line - parameters to pass to ArgumentParser.add_argument +# converter - (optional) Function to call if conversion is required +# if this configuration entry changes +config_entries = { + 'prefix_len': { + 'implicit': 1, + 'default': 2, + 'command-line': { + 'help': 'Length of cache file name used as subdirectory prefix', + 'metavar': '', + 'type': int + }, + 'converter': rearrange_cache_entries + } +} + + +def main(): + parser = argparse.ArgumentParser( + description='Modify the configuration of an scons cache directory', + epilog=''' + Unspecified options will not be changed unless they are not + set at all, in which case they are set to an appropriate default. + ''') + + parser.add_argument('cache-dir', help='Path to scons cache directory') + for param in config_entries: + parser.add_argument('--' + param.replace('_', '-'), + **config_entries[param]['command-line']) + parser.add_argument('--version', + action='version', + version='%(prog)s 1.0') + parser.add_argument('--show', + action="store_true", + help="show current configuration") + + # Get the command line as a dict without any of the unspecified entries. + args = dict([x for x in vars(parser.parse_args()).items() if x[1]]) + + # It seems somewhat strange to me, but positional arguments don't get the - + # in the name changed to _, whereas optional arguments do... + cache = args['cache-dir'] + if not os.path.isdir(cache): + raise RuntimeError("There is no cache directory named %s" % cache) + os.chdir(cache) + del args['cache-dir'] + + if not os.path.exists('config'): + # old config dirs did not have a 'config' file. Try to update. + # Validate the only files in the directory are directories 0-9, a-f + expected = ['{:X}'.format(x) for x in range(0, 16)] + if not set(os.listdir('.')).issubset(expected): + raise RuntimeError( + "%s does not look like a valid version 1 cache directory" % cache) + config = dict() + else: + with open('config') as conf: + config = json.load(conf) + + if args.get('show', None): + print("Current configuration in '%s':" % cache) + print(json.dumps(config, sort_keys=True, + indent=4, separators=(',', ': '))) + # in case of the show argument, emit some stats as well + file_count = 0 + for _, _, files in os.walk('.'): + file_count += len(files) + if file_count: # skip config file if it exists + file_count -= 1 + print("Cache contains %s files" % file_count) + del args['show'] + + # Find any keys that are not currently set but should be + for key in config_entries: + if key not in config: + if 'implicit' in config_entries[key]: + config[key] = config_entries[key]['implicit'] + else: + config[key] = config_entries[key]['default'] + if key not in args: + args[key] = config_entries[key]['default'] + + # Now go through each entry in args to see if it changes an existing config + # setting. + for key in args: + if args[key] != config[key]: + if 'converter' in config_entries[key]: + config_entries[key]['converter'](config[key], args[key]) + config[key] = args[key] + + # and write the updated config file + with open('config', 'w') as conf: + json.dump(config, conf) + +if __name__ == "__main__": + main() \ No newline at end of file -- cgit v0.12 From a51455a5b9d359f7ea5bc578d776f1aedb952d77 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 21 Mar 2020 17:36:38 -0700 Subject: comment out packaging part of build --- SConstruct | 1260 ++++++++++++++++++++++++++++---------------------------- doc/SConscript | 2 +- 2 files changed, 631 insertions(+), 631 deletions(-) diff --git a/SConstruct b/SConstruct index 6a11bca..f696499 100644 --- a/SConstruct +++ b/SConstruct @@ -200,462 +200,462 @@ Version_values = [Value(command_line.version), Value(command_line.build_id)] # separate packages. # -from distutils.sysconfig import get_python_lib - -python_scons = { - 'pkg': 'python-' + project, - 'src_subdir': 'engine', - 'inst_subdir': get_python_lib(), - - 'files': ['LICENSE.txt', - 'README.txt', - 'setup.cfg', - 'setup.py', - ], - - 'filemap': { - 'LICENSE.txt': '../LICENSE.txt' - }, - - 'buildermap': {}, - - 'explicit_deps': { - 'SCons/__init__.py': Version_values, - }, -} - -scons_script = { - 'pkg': project + '-script', - 'src_subdir': 'script', - 'inst_subdir': 'bin', - - 'files': [ - 'LICENSE.txt', - 'README.txt', - 'setup.cfg', - 'setup.py', - ], - - 'filemap': { - 'LICENSE.txt': '../LICENSE.txt', - 'scons': 'scons.py', - 'sconsign': 'sconsign.py', - 'scons-time': 'scons-time.py', - 'scons-configure-cache': 'scons-configure-cache.py', - }, - - 'buildermap': {}, - - 'explicit_deps': { - 'scons': Version_values, - 'sconsign': Version_values, - }, -} - -scons = { - 'pkg': project, - - 'files': [ - 'CHANGES.txt', - 'LICENSE.txt', - 'README.txt', - 'RELEASE.txt', - 'scons.1', - 'sconsign.1', - 'scons-time.1', - 'script/scons.bat', - 'setup.cfg', - 'setup.py', - ], - - 'filemap': { - 'scons.1': '$BUILDDIR/doc/man/scons.1', - 'sconsign.1': '$BUILDDIR/doc/man/sconsign.1', - 'scons-time.1': '$BUILDDIR/doc/man/scons-time.1', - }, - - 'buildermap': { - 'scons.1': env.SOElim, - 'sconsign.1': env.SOElim, - 'scons-time.1': env.SOElim, - }, - - 'subpkgs': [python_scons, scons_script], - - 'subinst_dirs': { - 'python-' + project: python_project_subinst_dir, - project + '-script': project_script_subinst_dir, - }, -} - -scripts = ['scons', 'sconsign', 'scons-time', 'scons-configure-cache'] - -src_deps = [] -src_files = [] - -for p in [scons]: - # - # Initialize variables with the right directories for this package. - # - pkg = p['pkg'] - pkg_version = "%s-%s" % (pkg, command_line.version) - - src = 'src' - if 'src_subdir' in p: - src = os.path.join(src, p['src_subdir']) - - build = os.path.join(command_line.build_dir, pkg) - - tar_gz = os.path.join(build, 'dist', "%s.tar.gz" % pkg_version) - platform_tar_gz = os.path.join(build, - 'dist', - "%s.%s.tar.gz" % (pkg_version, platform)) - zip = os.path.join(build, 'dist', "%s.zip" % pkg_version) - platform_zip = os.path.join(build, - 'dist', - "%s.%s.zip" % (pkg_version, platform)) - - # - # Update the environment with the relevant information - # for this package. - # - # We can get away with calling setup.py using a directory path - # like this because we put a preamble in it that will chdir() - # to the directory in which setup.py exists. - # - setup_py = os.path.join(build, 'setup.py') - env.Replace(PKG=pkg, - PKG_VERSION=pkg_version, - SETUP_PY='"%s"' % setup_py) - Local(setup_py) - - # - # Read up the list of source files from our MANIFEST.in. - # This list should *not* include LICENSE.txt, MANIFEST, - # README.txt, or setup.py. Make a copy of the list for the - # destination files. - # - manifest_in = File(os.path.join(src, 'MANIFEST.in')).rstr() - src_files = bootstrap.parseManifestLines(src, manifest_in) - raw_files = src_files[:] - dst_files = src_files[:] - - MANIFEST_in_list = [] - - if 'subpkgs' in p: - # - # This package includes some sub-packages. Read up their - # MANIFEST.in files, and add them to our source and destination - # file lists, modifying them as appropriate to add the - # specified subdirs. - # - for sp in p['subpkgs']: - ssubdir = sp['src_subdir'] - isubdir = p['subinst_dirs'][sp['pkg']] - - MANIFEST_in = File(os.path.join(src, ssubdir, 'MANIFEST.in')).rstr() - MANIFEST_in_list.append(MANIFEST_in) - files = bootstrap.parseManifestLines(os.path.join(src, ssubdir), MANIFEST_in) - - raw_files.extend(files) - src_files.extend([os.path.join(ssubdir, x) for x in files]) - - files = [os.path.join(isubdir, x) for x in files] - dst_files.extend(files) - for k, f in sp['filemap'].items(): - if f: - k = os.path.join(ssubdir, k) - p['filemap'][k] = os.path.join(ssubdir, f) - for f, deps in sp['explicit_deps'].items(): - f = os.path.join(build, ssubdir, f) - env.Depends(f, deps) - - # - # Now that we have the "normal" source files, add those files - # that are standard for each distribution. Note that we don't - # add these to dst_files, because they don't get installed. - # And we still have the MANIFEST to add. - # - src_files.extend(p['files']) - - # - # Now run everything in src_file through the sed command we - # concocted to expand __FILE__, __VERSION__, etc. - # - for b in src_files: - s = p['filemap'].get(b, b) - if not s[0] == '$' and not os.path.isabs(s): - s = os.path.join(src, s) - - builder = p['buildermap'].get(b, env.SCons_revision) - x = builder(os.path.join(build, b), s) - - Local(x) - - # - # NOW, finally, we can create the MANIFEST, which we do - # by having Python spit out the contents of the src_files - # array we've carefully created. After we've added - # MANIFEST itself to the array, of course. - # - src_files.append("MANIFEST") - MANIFEST_in_list.append(os.path.join(src, 'MANIFEST.in')) - - - def write_src_files(target, source, **kw): - global src_files - src_files.sort() - with open(str(target[0]), 'w') as f: - for file in src_files: - f.write(file + "\n") - return 0 - - - env.Command(os.path.join(build, 'MANIFEST'), - MANIFEST_in_list, - write_src_files) - - # - # Now go through and arrange to create whatever packages we can. - # - build_src_files = [os.path.join(build, x) for x in src_files] - Local(*build_src_files) - - distutils_formats = [] - distutils_targets = [] - dist_distutils_targets = [] - - for target in distutils_targets: - dist_target = env.Install('$DISTDIR', target) - AddPostAction(dist_target, Chmod(dist_target, 0o644)) - dist_distutils_targets += dist_target - - if not gzip: - print("gzip not found in %s; skipping .tar.gz package for %s." % (os.environ['PATH'], pkg)) - else: - - distutils_formats.append('gztar') - - src_deps.append(tar_gz) - - distutils_targets.extend([tar_gz, platform_tar_gz]) - - dist_tar_gz = env.Install('$DISTDIR', tar_gz) - dist_platform_tar_gz = env.Install('$DISTDIR', platform_tar_gz) - Local(dist_tar_gz, dist_platform_tar_gz) - AddPostAction(dist_tar_gz, Chmod(dist_tar_gz, 0o644)) - AddPostAction(dist_platform_tar_gz, Chmod(dist_platform_tar_gz, 0o644)) - - # - # Unpack the tar.gz archive created by the distutils into - # build/unpack-tar-gz/scons-{version}. - # - # We'd like to replace the last three lines with the following: - # - # tar zxf $SOURCES -C $UNPACK_TAR_GZ_DIR - # - # but that gives heartburn to Cygwin's tar, so work around it - # with separate zcat-tar-rm commands. - # - unpack_tar_gz_files = [os.path.join(unpack_tar_gz_dir, pkg_version, x) - for x in src_files] - env.Command(unpack_tar_gz_files, dist_tar_gz, [ - Delete(os.path.join(unpack_tar_gz_dir, pkg_version)), - "$ZCAT $SOURCES > .temp", - "tar xf .temp -C $UNPACK_TAR_GZ_DIR", - Delete(".temp"), - ]) - - # - # Run setup.py in the unpacked subdirectory to "install" everything - # into our build/test subdirectory. The runtest.py script will set - # PYTHONPATH so that the tests only look under build/test-{package}, - # and under testing/framework (for the testing modules TestCmd.py, TestSCons.py, - # etc.). This makes sure that our tests pass with what - # we really packaged, not because of something hanging around in - # the development directory. - # - # We can get away with calling setup.py using a directory path - # like this because we put a preamble in it that will chdir() - # to the directory in which setup.py exists. - # - dfiles = [os.path.join(test_tar_gz_dir, x) for x in dst_files] - env.Command(dfiles, unpack_tar_gz_files, [ - Delete(os.path.join(unpack_tar_gz_dir, pkg_version, 'build')), - Delete("$TEST_TAR_GZ_DIR"), - '$PYTHON $PYTHONFLAGS "%s" install "--prefix=$TEST_TAR_GZ_DIR" --standalone-lib' % \ - os.path.join(unpack_tar_gz_dir, pkg_version, 'setup.py'), - ]) - - # - # Generate portage files for submission to Gentoo Linux. - # - gentoo = os.path.join(build, 'gentoo') - ebuild = os.path.join(gentoo, 'scons-%s.ebuild' % command_line.version) - digest = os.path.join(gentoo, 'files', 'digest-scons-%s' % command_line.version) - env.Command(ebuild, os.path.join('gentoo', 'scons.ebuild.in'), SCons_revision) - - - def Digestify(target, source, env): - import hashlib - src = source[0].rfile() - with open(str(src), 'rb') as f: - contents = f.read() - m = hashlib.md5() - m.update(contents) - sig = m.hexdigest() - bytes = os.stat(str(src))[6] - with open(str(target[0]), 'w') as f: - f.write("MD5 %s %s %d\n" % (sig, src.name, bytes)) - - - env.Command(digest, tar_gz, Digestify) - - if not zipit: - print("zip not found; skipping .zip package for %s." % pkg) - else: - - distutils_formats.append('zip') - - src_deps.append(zip) - - distutils_targets.extend([zip, platform_zip]) - - dist_zip = env.Install('$DISTDIR', zip) - dist_platform_zip = env.Install('$DISTDIR', platform_zip) - Local(dist_zip, dist_platform_zip) - AddPostAction(dist_zip, Chmod(dist_zip, 0o644)) - AddPostAction(dist_platform_zip, Chmod(dist_platform_zip, 0o644)) - - # - # Unpack the zip archive created by the distutils into - # build/unpack-zip/scons-{version}. - # - unpack_zip_files = [os.path.join(unpack_zip_dir, pkg_version, x) - for x in src_files] - - env.Command(unpack_zip_files, dist_zip, [ - Delete(os.path.join(unpack_zip_dir, pkg_version)), - unzipit, - ]) - - # - # Run setup.py in the unpacked subdirectory to "install" everything - # into our build/test subdirectory. The runtest.py script will set - # PYTHONPATH so that the tests only look under build/test-{package}, - # and under testing/framework (for the testing modules TestCmd.py, TestSCons.py, - # etc.). This makes sure that our tests pass with what - # we really packaged, not because of something hanging around in - # the development directory. - # - # We can get away with calling setup.py using a directory path - # like this because we put a preamble in it that will chdir() - # to the directory in which setup.py exists. - # - dfiles = [os.path.join(test_zip_dir, x) for x in dst_files] - env.Command(dfiles, unpack_zip_files, [ - Delete(os.path.join(unpack_zip_dir, pkg_version, 'build')), - Delete("$TEST_ZIP_DIR"), - '$PYTHON $PYTHONFLAGS "%s" install "--prefix=$TEST_ZIP_DIR" --standalone-lib' % \ - os.path.join(unpack_zip_dir, pkg_version, 'setup.py'), - ]) - - # - # Use the Python distutils to generate the appropriate packages. - # - commands = [ - Delete(os.path.join(build, 'build', 'lib')), - Delete(os.path.join(build, 'build', 'scripts')), - ] - - if distutils_formats: - commands.append(Delete(os.path.join(build, - 'build', - 'bdist.' + platform, - 'dumb'))) - for format in distutils_formats: - commands.append("$PYTHON $PYTHONFLAGS $SETUP_PY bdist_dumb -f %s" % format) - - commands.append("$PYTHON $PYTHONFLAGS $SETUP_PY sdist --formats=%s" % \ - ','.join(distutils_formats)) - - env.Command(distutils_targets, build_src_files, commands) - - # - # Now create local packages for people who want to let people - # build their SCons-buildable packages without having to - # install SCons. - # - s_l_v = '%s-local-%s' % (pkg, command_line.version) - - local = pkg + '-local' - build_dir_local = os.path.join(command_line.build_dir, local) - build_dir_local_slv = os.path.join(command_line.build_dir, local, s_l_v) - - dist_local_tar_gz = os.path.join("$DISTDIR/%s.tar.gz" % s_l_v) - dist_local_zip = os.path.join("$DISTDIR/%s.zip" % s_l_v) - AddPostAction(dist_local_tar_gz, Chmod(dist_local_tar_gz, 0o644)) - AddPostAction(dist_local_zip, Chmod(dist_local_zip, 0o644)) - - commands = [ - Delete(build_dir_local), - '$PYTHON $PYTHONFLAGS $SETUP_PY install "--install-script=%s" "--install-lib=%s" --no-install-man --no-compile --standalone-lib --no-version-script' % \ - (build_dir_local, build_dir_local_slv), - ] - - for script in scripts: - # add .py extension for scons-local scripts on non-windows platforms - if is_windows(): - break - local_script = os.path.join(build_dir_local, script) - commands.append(Move(local_script + '.py', local_script)) - - rf = [x for x in raw_files if not x in scripts] - rf = [os.path.join(s_l_v, x) for x in rf] - for script in scripts: - rf.append("%s.py" % script) - local_targets = [os.path.join(build_dir_local, x) for x in rf] - - env.Command(local_targets, build_src_files, commands) - - scons_LICENSE = os.path.join(build_dir_local, 'scons-LICENSE') - l = env.SCons_revision(scons_LICENSE, 'LICENSE-local') - local_targets.append(l) - Local(l) - - scons_README = os.path.join(build_dir_local, 'scons-README') - l = env.SCons_revision(scons_README, 'README-local') - local_targets.append(l) - Local(l) - - if gzip: - if is_windows(): - # avoid problem with tar interpreting c:/ as a remote machine - tar_cargs = '-cz --force-local -f' - else: - tar_cargs = '-czf' - env.Command(dist_local_tar_gz, - local_targets, - "cd %s && tar %s $( ${TARGET.abspath} $) *" % (build_dir_local, tar_cargs)) - - unpack_targets = [os.path.join(test_local_tar_gz_dir, x) for x in rf] - commands = [Delete(test_local_tar_gz_dir), - Mkdir(test_local_tar_gz_dir), - "cd %s && tar xzf $( ${SOURCE.abspath} $)" % test_local_tar_gz_dir] - - env.Command(unpack_targets, dist_local_tar_gz, commands) - - if zipit: - env.Command(dist_local_zip, local_targets, zipit, - CD=build_dir_local, PSV='.') - - unpack_targets = [os.path.join(test_local_zip_dir, x) for x in rf] - commands = [Delete(test_local_zip_dir), - Mkdir(test_local_zip_dir), - unzipit] - - env.Command(unpack_targets, dist_local_zip, unzipit, - UNPACK_ZIP_DIR=test_local_zip_dir) +# from distutils.sysconfig import get_python_lib +# +# python_scons = { +# 'pkg': 'python-' + project, +# 'src_subdir': 'engine', +# 'inst_subdir': get_python_lib(), +# +# 'files': ['LICENSE.txt', +# 'README.txt', +# 'setup.cfg', +# 'setup.py', +# ], +# +# 'filemap': { +# 'LICENSE.txt': '../LICENSE.txt' +# }, +# +# 'buildermap': {}, +# +# 'explicit_deps': { +# 'SCons/__init__.py': Version_values, +# }, +# } + +# scons_script = { +# 'pkg': project + '-script', +# 'src_subdir': 'script', +# 'inst_subdir': 'bin', +# +# 'files': [ +# 'LICENSE.txt', +# 'README.txt', +# 'setup.cfg', +# 'setup.py', +# ], +# +# 'filemap': { +# 'LICENSE.txt': '../LICENSE.txt', +# 'scons': 'scons.py', +# 'sconsign': 'sconsign.py', +# 'scons-time': 'scons-time.py', +# 'scons-configure-cache': 'scons-configure-cache.py', +# }, +# +# 'buildermap': {}, +# +# 'explicit_deps': { +# 'scons': Version_values, +# 'sconsign': Version_values, +# }, +# } + +# scons = { +# 'pkg': project, +# +# 'files': [ +# 'CHANGES.txt', +# 'LICENSE.txt', +# 'README.txt', +# 'RELEASE.txt', +# 'scons.1', +# 'sconsign.1', +# 'scons-time.1', +# # 'script/scons.bat', +# 'setup.cfg', +# 'setup.py', +# ], +# +# 'filemap': { +# 'scons.1': '$BUILDDIR/doc/man/scons.1', +# 'sconsign.1': '$BUILDDIR/doc/man/sconsign.1', +# 'scons-time.1': '$BUILDDIR/doc/man/scons-time.1', +# }, +# +# 'buildermap': { +# 'scons.1': env.SOElim, +# 'sconsign.1': env.SOElim, +# 'scons-time.1': env.SOElim, +# }, +# +# 'subpkgs': [python_scons], +# +# 'subinst_dirs': { +# 'python-' + project: python_project_subinst_dir, +# project + '-script': project_script_subinst_dir, +# }, +# } +# +# scripts = ['scons', 'sconsign', 'scons-time', 'scons-configure-cache'] +# +# src_deps = [] +# src_files = [] +# +# for p in [scons]: +# # +# # Initialize variables with the right directories for this package. +# # +# pkg = p['pkg'] +# pkg_version = "%s-%s" % (pkg, command_line.version) +# +# src = 'src' +# if 'src_subdir' in p: +# src = os.path.join(src, p['src_subdir']) +# +# build = os.path.join(command_line.build_dir, pkg) +# +# tar_gz = os.path.join(build, 'dist', "%s.tar.gz" % pkg_version) +# platform_tar_gz = os.path.join(build, +# 'dist', +# "%s.%s.tar.gz" % (pkg_version, platform)) +# zip = os.path.join(build, 'dist', "%s.zip" % pkg_version) +# platform_zip = os.path.join(build, +# 'dist', +# "%s.%s.zip" % (pkg_version, platform)) +# +# # +# # Update the environment with the relevant information +# # for this package. +# # +# # We can get away with calling setup.py using a directory path +# # like this because we put a preamble in it that will chdir() +# # to the directory in which setup.py exists. +# # +# setup_py = os.path.join(build, 'setup.py') +# env.Replace(PKG=pkg, +# PKG_VERSION=pkg_version, +# SETUP_PY='"%s"' % setup_py) +# Local(setup_py) +# +# # +# # Read up the list of source files from our MANIFEST.in. +# # This list should *not* include LICENSE.txt, MANIFEST, +# # README.txt, or setup.py. Make a copy of the list for the +# # destination files. +# # +# manifest_in = File(os.path.join(src, 'MANIFEST.in')).rstr() +# src_files = bootstrap.parseManifestLines(src, manifest_in) +# raw_files = src_files[:] +# dst_files = src_files[:] +# +# MANIFEST_in_list = [] +# +# if 'subpkgs' in p: +# # +# # This package includes some sub-packages. Read up their +# # MANIFEST.in files, and add them to our source and destination +# # file lists, modifying them as appropriate to add the +# # specified subdirs. +# # +# for sp in p['subpkgs']: +# ssubdir = sp['src_subdir'] +# isubdir = p['subinst_dirs'][sp['pkg']] +# +# MANIFEST_in = File(os.path.join(src, ssubdir, 'MANIFEST.in')).rstr() +# MANIFEST_in_list.append(MANIFEST_in) +# files = bootstrap.parseManifestLines(os.path.join(src, ssubdir), MANIFEST_in) +# +# raw_files.extend(files) +# src_files.extend([os.path.join(ssubdir, x) for x in files]) +# +# files = [os.path.join(isubdir, x) for x in files] +# dst_files.extend(files) +# for k, f in sp['filemap'].items(): +# if f: +# k = os.path.join(ssubdir, k) +# p['filemap'][k] = os.path.join(ssubdir, f) +# for f, deps in sp['explicit_deps'].items(): +# f = os.path.join(build, ssubdir, f) +# env.Depends(f, deps) +# +# # +# # Now that we have the "normal" source files, add those files +# # that are standard for each distribution. Note that we don't +# # add these to dst_files, because they don't get installed. +# # And we still have the MANIFEST to add. +# # +# src_files.extend(p['files']) +# +# # +# # Now run everything in src_file through the sed command we +# # concocted to expand __FILE__, __VERSION__, etc. +# # +# for b in src_files: +# s = p['filemap'].get(b, b) +# if not s[0] == '$' and not os.path.isabs(s): +# s = os.path.join(src, s) +# +# builder = p['buildermap'].get(b, env.SCons_revision) +# x = builder(os.path.join(build, b), s) +# +# Local(x) +# +# # +# # NOW, finally, we can create the MANIFEST, which we do +# # by having Python spit out the contents of the src_files +# # array we've carefully created. After we've added +# # MANIFEST itself to the array, of course. +# # +# src_files.append("MANIFEST") +# MANIFEST_in_list.append(os.path.join(src, 'MANIFEST.in')) +# +# +# def write_src_files(target, source, **kw): +# global src_files +# src_files.sort() +# with open(str(target[0]), 'w') as f: +# for file in src_files: +# f.write(file + "\n") +# return 0 +# +# +# env.Command(os.path.join(build, 'MANIFEST'), +# MANIFEST_in_list, +# write_src_files) +# +# # +# # Now go through and arrange to create whatever packages we can. +# # +# build_src_files = [os.path.join(build, x) for x in src_files] +# Local(*build_src_files) +# +# distutils_formats = [] +# distutils_targets = [] +# dist_distutils_targets = [] +# +# for target in distutils_targets: +# dist_target = env.Install('$DISTDIR', target) +# AddPostAction(dist_target, Chmod(dist_target, 0o644)) +# dist_distutils_targets += dist_target +# +# if not gzip: +# print("gzip not found in %s; skipping .tar.gz package for %s." % (os.environ['PATH'], pkg)) +# else: +# +# distutils_formats.append('gztar') +# +# src_deps.append(tar_gz) +# +# distutils_targets.extend([tar_gz, platform_tar_gz]) +# +# dist_tar_gz = env.Install('$DISTDIR', tar_gz) +# dist_platform_tar_gz = env.Install('$DISTDIR', platform_tar_gz) +# Local(dist_tar_gz, dist_platform_tar_gz) +# AddPostAction(dist_tar_gz, Chmod(dist_tar_gz, 0o644)) +# AddPostAction(dist_platform_tar_gz, Chmod(dist_platform_tar_gz, 0o644)) +# +# # +# # Unpack the tar.gz archive created by the distutils into +# # build/unpack-tar-gz/scons-{version}. +# # +# # We'd like to replace the last three lines with the following: +# # +# # tar zxf $SOURCES -C $UNPACK_TAR_GZ_DIR +# # +# # but that gives heartburn to Cygwin's tar, so work around it +# # with separate zcat-tar-rm commands. +# # +# unpack_tar_gz_files = [os.path.join(unpack_tar_gz_dir, pkg_version, x) +# for x in src_files] +# env.Command(unpack_tar_gz_files, dist_tar_gz, [ +# Delete(os.path.join(unpack_tar_gz_dir, pkg_version)), +# "$ZCAT $SOURCES > .temp", +# "tar xf .temp -C $UNPACK_TAR_GZ_DIR", +# Delete(".temp"), +# ]) +# +# # +# # Run setup.py in the unpacked subdirectory to "install" everything +# # into our build/test subdirectory. The runtest.py script will set +# # PYTHONPATH so that the tests only look under build/test-{package}, +# # and under testing/framework (for the testing modules TestCmd.py, TestSCons.py, +# # etc.). This makes sure that our tests pass with what +# # we really packaged, not because of something hanging around in +# # the development directory. +# # +# # We can get away with calling setup.py using a directory path +# # like this because we put a preamble in it that will chdir() +# # to the directory in which setup.py exists. +# # +# dfiles = [os.path.join(test_tar_gz_dir, x) for x in dst_files] +# env.Command(dfiles, unpack_tar_gz_files, [ +# Delete(os.path.join(unpack_tar_gz_dir, pkg_version, 'build')), +# Delete("$TEST_TAR_GZ_DIR"), +# '$PYTHON $PYTHONFLAGS "%s" install "--prefix=$TEST_TAR_GZ_DIR" --standalone-lib' % \ +# os.path.join(unpack_tar_gz_dir, pkg_version, 'setup.py'), +# ]) +# +# # +# # Generate portage files for submission to Gentoo Linux. +# # +# gentoo = os.path.join(build, 'gentoo') +# ebuild = os.path.join(gentoo, 'scons-%s.ebuild' % command_line.version) +# digest = os.path.join(gentoo, 'files', 'digest-scons-%s' % command_line.version) +# env.Command(ebuild, os.path.join('gentoo', 'scons.ebuild.in'), SCons_revision) +# +# +# def Digestify(target, source, env): +# import hashlib +# src = source[0].rfile() +# with open(str(src), 'rb') as f: +# contents = f.read() +# m = hashlib.md5() +# m.update(contents) +# sig = m.hexdigest() +# bytes = os.stat(str(src))[6] +# with open(str(target[0]), 'w') as f: +# f.write("MD5 %s %s %d\n" % (sig, src.name, bytes)) +# +# +# env.Command(digest, tar_gz, Digestify) +# +# if not zipit: +# print("zip not found; skipping .zip package for %s." % pkg) +# else: +# +# distutils_formats.append('zip') +# +# src_deps.append(zip) +# +# distutils_targets.extend([zip, platform_zip]) +# +# dist_zip = env.Install('$DISTDIR', zip) +# dist_platform_zip = env.Install('$DISTDIR', platform_zip) +# Local(dist_zip, dist_platform_zip) +# AddPostAction(dist_zip, Chmod(dist_zip, 0o644)) +# AddPostAction(dist_platform_zip, Chmod(dist_platform_zip, 0o644)) +# +# # +# # Unpack the zip archive created by the distutils into +# # build/unpack-zip/scons-{version}. +# # +# unpack_zip_files = [os.path.join(unpack_zip_dir, pkg_version, x) +# for x in src_files] +# +# env.Command(unpack_zip_files, dist_zip, [ +# Delete(os.path.join(unpack_zip_dir, pkg_version)), +# unzipit, +# ]) +# +# # +# # Run setup.py in the unpacked subdirectory to "install" everything +# # into our build/test subdirectory. The runtest.py script will set +# # PYTHONPATH so that the tests only look under build/test-{package}, +# # and under testing/framework (for the testing modules TestCmd.py, TestSCons.py, +# # etc.). This makes sure that our tests pass with what +# # we really packaged, not because of something hanging around in +# # the development directory. +# # +# # We can get away with calling setup.py using a directory path +# # like this because we put a preamble in it that will chdir() +# # to the directory in which setup.py exists. +# # +# dfiles = [os.path.join(test_zip_dir, x) for x in dst_files] +# env.Command(dfiles, unpack_zip_files, [ +# Delete(os.path.join(unpack_zip_dir, pkg_version, 'build')), +# Delete("$TEST_ZIP_DIR"), +# '$PYTHON $PYTHONFLAGS "%s" install "--prefix=$TEST_ZIP_DIR" --standalone-lib' % \ +# os.path.join(unpack_zip_dir, pkg_version, 'setup.py'), +# ]) +# +# # +# # Use the Python distutils to generate the appropriate packages. +# # +# commands = [ +# Delete(os.path.join(build, 'build', 'lib')), +# Delete(os.path.join(build, 'build', 'scripts')), +# ] +# +# if distutils_formats: +# commands.append(Delete(os.path.join(build, +# 'build', +# 'bdist.' + platform, +# 'dumb'))) +# for format in distutils_formats: +# commands.append("$PYTHON $PYTHONFLAGS $SETUP_PY bdist_dumb -f %s" % format) +# +# commands.append("$PYTHON $PYTHONFLAGS $SETUP_PY sdist --formats=%s" % \ +# ','.join(distutils_formats)) +# +# env.Command(distutils_targets, build_src_files, commands) +# +# # +# # Now create local packages for people who want to let people +# # build their SCons-buildable packages without having to +# # install SCons. +# # +# s_l_v = '%s-local-%s' % (pkg, command_line.version) +# +# local = pkg + '-local' +# build_dir_local = os.path.join(command_line.build_dir, local) +# build_dir_local_slv = os.path.join(command_line.build_dir, local, s_l_v) +# +# dist_local_tar_gz = os.path.join("$DISTDIR/%s.tar.gz" % s_l_v) +# dist_local_zip = os.path.join("$DISTDIR/%s.zip" % s_l_v) +# AddPostAction(dist_local_tar_gz, Chmod(dist_local_tar_gz, 0o644)) +# AddPostAction(dist_local_zip, Chmod(dist_local_zip, 0o644)) +# +# commands = [ +# Delete(build_dir_local), +# '$PYTHON $PYTHONFLAGS $SETUP_PY install "--install-script=%s" "--install-lib=%s" --no-install-man --no-compile --standalone-lib --no-version-script' % \ +# (build_dir_local, build_dir_local_slv), +# ] +# +# for script in scripts: +# # add .py extension for scons-local scripts on non-windows platforms +# if is_windows(): +# break +# local_script = os.path.join(build_dir_local, script) +# commands.append(Move(local_script + '.py', local_script)) +# +# rf = [x for x in raw_files if not x in scripts] +# rf = [os.path.join(s_l_v, x) for x in rf] +# for script in scripts: +# rf.append("%s.py" % script) +# local_targets = [os.path.join(build_dir_local, x) for x in rf] +# +# env.Command(local_targets, build_src_files, commands) +# +# scons_LICENSE = os.path.join(build_dir_local, 'scons-LICENSE') +# l = env.SCons_revision(scons_LICENSE, 'LICENSE-local') +# local_targets.append(l) +# Local(l) +# +# scons_README = os.path.join(build_dir_local, 'scons-README') +# l = env.SCons_revision(scons_README, 'README-local') +# local_targets.append(l) +# Local(l) +# +# if gzip: +# if is_windows(): +# # avoid problem with tar interpreting c:/ as a remote machine +# tar_cargs = '-cz --force-local -f' +# else: +# tar_cargs = '-czf' +# env.Command(dist_local_tar_gz, +# local_targets, +# "cd %s && tar %s $( ${TARGET.abspath} $) *" % (build_dir_local, tar_cargs)) +# +# unpack_targets = [os.path.join(test_local_tar_gz_dir, x) for x in rf] +# commands = [Delete(test_local_tar_gz_dir), +# Mkdir(test_local_tar_gz_dir), +# "cd %s && tar xzf $( ${SOURCE.abspath} $)" % test_local_tar_gz_dir] +# +# env.Command(unpack_targets, dist_local_tar_gz, commands) +# +# if zipit: +# env.Command(dist_local_zip, local_targets, zipit, +# CD=build_dir_local, PSV='.') +# +# unpack_targets = [os.path.join(test_local_zip_dir, x) for x in rf] +# commands = [Delete(test_local_zip_dir), +# Mkdir(test_local_zip_dir), +# unzipit] +# +# env.Command(unpack_targets, dist_local_zip, unzipit, +# UNPACK_ZIP_DIR=test_local_zip_dir) # # @@ -667,179 +667,179 @@ Export('command_line', 'env', 'whereis', 'revaction') SConscript('doc/SConscript') +# # +# # If we're running in a Git working directory, pack up a complete +# # source archive from the project files and files in the change. +# # # -# If we're running in a Git working directory, pack up a complete -# source archive from the project files and files in the change. # - - -sfiles = [l.split()[-1] for l in command_line.git_status_lines] -if command_line.git_status_lines: - # slines = [l for l in git_status_lines if 'modified:' in l] - # sfiles = [l.split()[-1] for l in slines] - pass -else: - print("Not building in a Git tree; skipping building src package.") - -if sfiles: - remove_patterns = [ - '*.gitignore', - '*.hgignore', - 'www/*', - ] - - for p in remove_patterns: - sfiles = [s for s in sfiles if not fnmatch.fnmatch(s, p)] - - if sfiles: - ps = "%s-src" % project - psv = "%s-%s" % (ps, command_line.version) - b_ps = os.path.join(command_line.build_dir, ps) - b_psv = os.path.join(command_line.build_dir, psv) - b_psv_stamp = b_psv + '-stamp' - - src_tar_gz = os.path.join(command_line.build_dir, 'dist', '%s.tar.gz' % psv) - src_zip = os.path.join(command_line.build_dir, 'dist', '%s.zip' % psv) - - Local(src_tar_gz, src_zip) - - for file in sfiles: - if file.endswith('jpg') or file.endswith('png'): - # don't revision binary files. - env.Install(os.path.dirname(os.path.join(b_ps, file)), file) - else: - env.SCons_revision(os.path.join(b_ps, file), file) - - b_ps_files = [os.path.join(b_ps, x) for x in sfiles] - cmds = [ - Delete(b_psv), - Copy(b_psv, b_ps), - Touch("$TARGET"), - ] - - env.Command(b_psv_stamp, src_deps + b_ps_files, cmds) - - Local(*b_ps_files) - - if gzip: - env.Command(src_tar_gz, b_psv_stamp, - "tar cz${TAR_HFLAG} -f $TARGET -C build %s" % psv) - - # - # Unpack the archive into build/unpack/scons-{version}. - # - unpack_tar_gz_files = [os.path.join(unpack_tar_gz_dir, psv, x) - for x in sfiles] - - # - # We'd like to replace the last three lines with the following: - # - # tar zxf $SOURCES -C $UNPACK_TAR_GZ_DIR - # - # but that gives heartburn to Cygwin's tar, so work around it - # with separate zcat-tar-rm commands. - env.Command(unpack_tar_gz_files, src_tar_gz, [ - Delete(os.path.join(unpack_tar_gz_dir, psv)), - "$ZCAT $SOURCES > .temp", - "tar xf .temp -C $UNPACK_TAR_GZ_DIR", - Delete(".temp"), - ]) - - # - # Run setup.py in the unpacked subdirectory to "install" everything - # into our build/test subdirectory. The runtest.py script will set - # PYTHONPATH so that the tests only look under build/test-{package}, - # and under testing/framework (for the testing modules TestCmd.py, - # TestSCons.py, etc.). This makes sure that our tests pass with - # what we really packaged, not because of something hanging around - # in the development directory. - # - # We can get away with calling setup.py using a directory path - # like this because we put a preamble in it that will chdir() - # to the directory in which setup.py exists. - # - dfiles = [os.path.join(test_src_tar_gz_dir, x) for x in dst_files] - scons_lib_dir = os.path.join(unpack_tar_gz_dir, psv, 'src', 'engine') - ENV = env.Dictionary('ENV').copy() - ENV['SCONS_LIB_DIR'] = scons_lib_dir - ENV['USERNAME'] = command_line.developer - env.Command(dfiles, unpack_tar_gz_files, - [ - Delete(os.path.join(unpack_tar_gz_dir, - psv, - 'build', - 'scons', - 'build')), - Delete("$TEST_SRC_TAR_GZ_DIR"), - 'cd "%s" && $PYTHON $PYTHONFLAGS "%s" "%s" VERSION="$VERSION"' % \ - (os.path.join(unpack_tar_gz_dir, psv), - os.path.join('src', 'script', 'scons.py'), - os.path.join('build', 'scons')), - '$PYTHON $PYTHONFLAGS "%s" install "--prefix=$TEST_SRC_TAR_GZ_DIR" --standalone-lib' % \ - os.path.join(unpack_tar_gz_dir, - psv, - 'build', - 'scons', - 'setup.py'), - ], - ENV=ENV) - - if zipit: - env.Command(src_zip, b_psv_stamp, zipit, CD='build', PSV=psv) - - # - # Unpack the archive into build/unpack/scons-{version}. - # - unpack_zip_files = [os.path.join(unpack_zip_dir, psv, x) - for x in sfiles] - - env.Command(unpack_zip_files, src_zip, [ - Delete(os.path.join(unpack_zip_dir, psv)), - unzipit - ]) - - # - # Run setup.py in the unpacked subdirectory to "install" everything - # into our build/test subdirectory. The runtest.py script will set - # PYTHONPATH so that the tests only look under build/test-{package}, - # and under testing/framework (for the testing modules TestCmd.py, - # TestSCons.py, etc.). This makes sure that our tests pass with - # what we really packaged, not because of something hanging - # around in the development directory. - # - # We can get away with calling setup.py using a directory path - # like this because we put a preamble in it that will chdir() - # to the directory in which setup.py exists. - # - dfiles = [os.path.join(test_src_zip_dir, x) for x in dst_files] - scons_lib_dir = os.path.join(unpack_zip_dir, psv, 'src', 'engine') - ENV = env.Dictionary('ENV').copy() - ENV['SCONS_LIB_DIR'] = scons_lib_dir - ENV['USERNAME'] = command_line.developer - env.Command(dfiles, unpack_zip_files, - [ - Delete(os.path.join(unpack_zip_dir, - psv, - 'build', - 'scons', - 'build')), - Delete("$TEST_SRC_ZIP_DIR"), - 'cd "%s" && $PYTHON $PYTHONFLAGS "%s" "%s" VERSION="$VERSION"' % \ - (os.path.join(unpack_zip_dir, psv), - os.path.join('src', 'script', 'scons.py'), - os.path.join('build', 'scons')), - '$PYTHON $PYTHONFLAGS "%s" install "--prefix=$TEST_SRC_ZIP_DIR" --standalone-lib' % \ - os.path.join(unpack_zip_dir, - psv, - 'build', - 'scons', - 'setup.py'), - ], - ENV=ENV) - -for pf, help_text in packaging_flavors: - Alias(pf, [ - os.path.join(command_line.build_dir, 'test-' + pf), - os.path.join(command_line.build_dir, 'testing/framework'), - os.path.join(command_line.build_dir, 'runtest.py'), - ]) +# sfiles = [l.split()[-1] for l in command_line.git_status_lines] +# if command_line.git_status_lines: +# # slines = [l for l in git_status_lines if 'modified:' in l] +# # sfiles = [l.split()[-1] for l in slines] +# pass +# else: +# print("Not building in a Git tree; skipping building src package.") +# +# if sfiles: +# remove_patterns = [ +# '*.gitignore', +# '*.hgignore', +# 'www/*', +# ] +# +# for p in remove_patterns: +# sfiles = [s for s in sfiles if not fnmatch.fnmatch(s, p)] +# +# if sfiles: +# ps = "%s-src" % project +# psv = "%s-%s" % (ps, command_line.version) +# b_ps = os.path.join(command_line.build_dir, ps) +# b_psv = os.path.join(command_line.build_dir, psv) +# b_psv_stamp = b_psv + '-stamp' +# +# src_tar_gz = os.path.join(command_line.build_dir, 'dist', '%s.tar.gz' % psv) +# src_zip = os.path.join(command_line.build_dir, 'dist', '%s.zip' % psv) +# +# Local(src_tar_gz, src_zip) +# +# for file in sfiles: +# if file.endswith('jpg') or file.endswith('png'): +# # don't revision binary files. +# env.Install(os.path.dirname(os.path.join(b_ps, file)), file) +# else: +# env.SCons_revision(os.path.join(b_ps, file), file) +# +# b_ps_files = [os.path.join(b_ps, x) for x in sfiles] +# cmds = [ +# Delete(b_psv), +# Copy(b_psv, b_ps), +# Touch("$TARGET"), +# ] +# +# env.Command(b_psv_stamp, src_deps + b_ps_files, cmds) +# +# Local(*b_ps_files) +# +# if gzip: +# env.Command(src_tar_gz, b_psv_stamp, +# "tar cz${TAR_HFLAG} -f $TARGET -C build %s" % psv) +# +# # +# # Unpack the archive into build/unpack/scons-{version}. +# # +# unpack_tar_gz_files = [os.path.join(unpack_tar_gz_dir, psv, x) +# for x in sfiles] +# +# # +# # We'd like to replace the last three lines with the following: +# # +# # tar zxf $SOURCES -C $UNPACK_TAR_GZ_DIR +# # +# # but that gives heartburn to Cygwin's tar, so work around it +# # with separate zcat-tar-rm commands. +# env.Command(unpack_tar_gz_files, src_tar_gz, [ +# Delete(os.path.join(unpack_tar_gz_dir, psv)), +# "$ZCAT $SOURCES > .temp", +# "tar xf .temp -C $UNPACK_TAR_GZ_DIR", +# Delete(".temp"), +# ]) +# +# # +# # Run setup.py in the unpacked subdirectory to "install" everything +# # into our build/test subdirectory. The runtest.py script will set +# # PYTHONPATH so that the tests only look under build/test-{package}, +# # and under testing/framework (for the testing modules TestCmd.py, +# # TestSCons.py, etc.). This makes sure that our tests pass with +# # what we really packaged, not because of something hanging around +# # in the development directory. +# # +# # We can get away with calling setup.py using a directory path +# # like this because we put a preamble in it that will chdir() +# # to the directory in which setup.py exists. +# # +# dfiles = [os.path.join(test_src_tar_gz_dir, x) for x in dst_files] +# scons_lib_dir = os.path.join(unpack_tar_gz_dir, psv, 'src', 'engine') +# ENV = env.Dictionary('ENV').copy() +# ENV['SCONS_LIB_DIR'] = scons_lib_dir +# ENV['USERNAME'] = command_line.developer +# env.Command(dfiles, unpack_tar_gz_files, +# [ +# Delete(os.path.join(unpack_tar_gz_dir, +# psv, +# 'build', +# 'scons', +# 'build')), +# Delete("$TEST_SRC_TAR_GZ_DIR"), +# 'cd "%s" && $PYTHON $PYTHONFLAGS "%s" "%s" VERSION="$VERSION"' % \ +# (os.path.join(unpack_tar_gz_dir, psv), +# os.path.join('src', 'script', 'scons.py'), +# os.path.join('build', 'scons')), +# '$PYTHON $PYTHONFLAGS "%s" install "--prefix=$TEST_SRC_TAR_GZ_DIR" --standalone-lib' % \ +# os.path.join(unpack_tar_gz_dir, +# psv, +# 'build', +# 'scons', +# 'setup.py'), +# ], +# ENV=ENV) +# +# if zipit: +# env.Command(src_zip, b_psv_stamp, zipit, CD='build', PSV=psv) +# +# # +# # Unpack the archive into build/unpack/scons-{version}. +# # +# unpack_zip_files = [os.path.join(unpack_zip_dir, psv, x) +# for x in sfiles] +# +# env.Command(unpack_zip_files, src_zip, [ +# Delete(os.path.join(unpack_zip_dir, psv)), +# unzipit +# ]) +# +# # +# # Run setup.py in the unpacked subdirectory to "install" everything +# # into our build/test subdirectory. The runtest.py script will set +# # PYTHONPATH so that the tests only look under build/test-{package}, +# # and under testing/framework (for the testing modules TestCmd.py, +# # TestSCons.py, etc.). This makes sure that our tests pass with +# # what we really packaged, not because of something hanging +# # around in the development directory. +# # +# # We can get away with calling setup.py using a directory path +# # like this because we put a preamble in it that will chdir() +# # to the directory in which setup.py exists. +# # +# dfiles = [os.path.join(test_src_zip_dir, x) for x in dst_files] +# scons_lib_dir = os.path.join(unpack_zip_dir, psv, 'src', 'engine') +# ENV = env.Dictionary('ENV').copy() +# ENV['SCONS_LIB_DIR'] = scons_lib_dir +# ENV['USERNAME'] = command_line.developer +# env.Command(dfiles, unpack_zip_files, +# [ +# Delete(os.path.join(unpack_zip_dir, +# psv, +# 'build', +# 'scons', +# 'build')), +# Delete("$TEST_SRC_ZIP_DIR"), +# 'cd "%s" && $PYTHON $PYTHONFLAGS "%s" "%s" VERSION="$VERSION"' % \ +# (os.path.join(unpack_zip_dir, psv), +# os.path.join('src', 'script', 'scons.py'), +# os.path.join('build', 'scons')), +# '$PYTHON $PYTHONFLAGS "%s" install "--prefix=$TEST_SRC_ZIP_DIR" --standalone-lib' % \ +# os.path.join(unpack_zip_dir, +# psv, +# 'build', +# 'scons', +# 'setup.py'), +# ], +# ENV=ENV) +# +# for pf, help_text in packaging_flavors: +# Alias(pf, [ +# os.path.join(command_line.build_dir, 'test-' + pf), +# os.path.join(command_line.build_dir, 'testing/framework'), +# os.path.join(command_line.build_dir, 'runtest.py'), +# ]) diff --git a/doc/SConscript b/doc/SConscript index 4e9fb74..df68ecc 100644 --- a/doc/SConscript +++ b/doc/SConscript @@ -76,7 +76,7 @@ tar_deps = [] tar_list = [] orig_env = env -env = orig_env.Clone(SCONS_PY = File('#src/script/scons.py').rfile()) +env = orig_env.Clone(SCONS_PY = File('#/scripts/scons.py').rfile()) # # --- Helpers --- -- cgit v0.12 From 111837ba11bdf49703e9d6bdc5e79f0231a88095 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 21 Mar 2020 17:43:51 -0700 Subject: Fix doc sconscript --- doc/SConscript | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/SConscript b/doc/SConscript index df68ecc..8b22b3b 100644 --- a/doc/SConscript +++ b/doc/SConscript @@ -562,4 +562,4 @@ if tar_deps: Local(t) Alias('doc', t) else: - Alias('doc', os.path.join(build_dir, 'doc')) + Alias('doc', os.path.join(command_line.build_dir, 'doc')) -- cgit v0.12 From 349d79172a60b2840a17c03456be93560ec974e0 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 21 Mar 2020 18:11:03 -0700 Subject: Fix scripts dirname typo --- runtest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtest.py b/runtest.py index 2ee4476..65b2179 100755 --- a/runtest.py +++ b/runtest.py @@ -555,7 +555,7 @@ else: scons_runtest_dir = base if not external: - scons_script_dir = sd or os.path.join(base, 'script') + scons_script_dir = sd or os.path.join(base, 'scripts') scons_lib_dir = ld or os.path.join(base, 'src', 'engine') else: scons_script_dir = sd or '' -- cgit v0.12 From 4ace04ea384de1e76126c4a6dfae8c8fbe2b3db8 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 21 Mar 2020 19:22:33 -0700 Subject: Fix scons-time tests. Remove subversion test --- bin/scons-time.py | 1480 --------------------------------- scripts/scons-time.py | 1480 +++++++++++++++++++++++++++++++++ test/scons-time/run/archive/dir.py | 1 + test/scons-time/run/config/targets.py | 2 +- test/scons-time/run/option/quiet.py | 2 +- test/scons-time/run/option/verbose.py | 2 +- test/scons-time/run/subversion.py | 81 -- testing/framework/TestSCons_time.py | 8 +- 8 files changed, 1488 insertions(+), 1568 deletions(-) delete mode 100644 bin/scons-time.py create mode 100644 scripts/scons-time.py delete mode 100644 test/scons-time/run/subversion.py diff --git a/bin/scons-time.py b/bin/scons-time.py deleted file mode 100644 index e4dd863..0000000 --- a/bin/scons-time.py +++ /dev/null @@ -1,1480 +0,0 @@ -#!/usr/bin/env python -# -# scons-time - run SCons timings and collect statistics -# -# A script for running a configuration through SCons with a standard -# set of invocations to collect timing and memory statistics and to -# capture the results in a consistent set of output files for display -# and analysis. -# - -# -# __COPYRIGHT__ -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - -import getopt -import glob -import os -import re -import shutil -import sys -import tempfile -import time -import subprocess - -def HACK_for_exec(cmd, *args): - """ - For some reason, Python won't allow an exec() within a function - that also declares an internal function (including lambda functions). - This function is a hack that calls exec() in a function with no - internal functions. - """ - if not args: exec(cmd) - elif len(args) == 1: exec(cmd, args[0]) - else: exec(cmd, args[0], args[1]) - -class Plotter(object): - def increment_size(self, largest): - """ - Return the size of each horizontal increment line for a specified - maximum value. This returns a value that will provide somewhere - between 5 and 9 horizontal lines on the graph, on some set of - boundaries that are multiples of 10/100/1000/etc. - """ - i = largest // 5 - if not i: - return largest - multiplier = 1 - while i >= 10: - i = i // 10 - multiplier = multiplier * 10 - return i * multiplier - - def max_graph_value(self, largest): - # Round up to next integer. - largest = int(largest) + 1 - increment = self.increment_size(largest) - return ((largest + increment - 1) // increment) * increment - -class Line(object): - def __init__(self, points, type, title, label, comment, fmt="%s %s"): - self.points = points - self.type = type - self.title = title - self.label = label - self.comment = comment - self.fmt = fmt - - def print_label(self, inx, x, y): - if self.label: - print('set label %s "%s" at %0.1f,%0.1f right' % (inx, self.label, x, y)) - - def plot_string(self): - if self.title: - title_string = 'title "%s"' % self.title - else: - title_string = 'notitle' - return "'-' %s with lines lt %s" % (title_string, self.type) - - def print_points(self, fmt=None): - if fmt is None: - fmt = self.fmt - if self.comment: - print('# %s' % self.comment) - for x, y in self.points: - # If y is None, it usually represents some kind of break - # in the line's index number. We might want to represent - # this some way rather than just drawing the line straight - # between the two points on either side. - if y is not None: - print(fmt % (x, y)) - print('e') - - def get_x_values(self): - return [ p[0] for p in self.points ] - - def get_y_values(self): - return [ p[1] for p in self.points ] - -class Gnuplotter(Plotter): - - def __init__(self, title, key_location): - self.lines = [] - self.title = title - self.key_location = key_location - - def line(self, points, type, title=None, label=None, comment=None, fmt='%s %s'): - if points: - line = Line(points, type, title, label, comment, fmt) - self.lines.append(line) - - def plot_string(self, line): - return line.plot_string() - - def vertical_bar(self, x, type, label, comment): - if self.get_min_x() <= x <= self.get_max_x(): - points = [(x, 0), (x, self.max_graph_value(self.get_max_y()))] - self.line(points, type, label, comment) - - def get_all_x_values(self): - result = [] - for line in self.lines: - result.extend(line.get_x_values()) - return [r for r in result if r is not None] - - def get_all_y_values(self): - result = [] - for line in self.lines: - result.extend(line.get_y_values()) - return [r for r in result if r is not None] - - def get_min_x(self): - try: - return self.min_x - except AttributeError: - try: - self.min_x = min(self.get_all_x_values()) - except ValueError: - self.min_x = 0 - return self.min_x - - def get_max_x(self): - try: - return self.max_x - except AttributeError: - try: - self.max_x = max(self.get_all_x_values()) - except ValueError: - self.max_x = 0 - return self.max_x - - def get_min_y(self): - try: - return self.min_y - except AttributeError: - try: - self.min_y = min(self.get_all_y_values()) - except ValueError: - self.min_y = 0 - return self.min_y - - def get_max_y(self): - try: - return self.max_y - except AttributeError: - try: - self.max_y = max(self.get_all_y_values()) - except ValueError: - self.max_y = 0 - return self.max_y - - def draw(self): - - if not self.lines: - return - - if self.title: - print('set title "%s"' % self.title) - print('set key %s' % self.key_location) - - min_y = self.get_min_y() - max_y = self.max_graph_value(self.get_max_y()) - incr = (max_y - min_y) / 10.0 - start = min_y + (max_y / 2.0) + (2.0 * incr) - position = [ start - (i * incr) for i in range(5) ] - - inx = 1 - for line in self.lines: - line.print_label(inx, line.points[0][0]-1, - position[(inx-1) % len(position)]) - inx += 1 - - plot_strings = [ self.plot_string(l) for l in self.lines ] - print('plot ' + ', \\\n '.join(plot_strings)) - - for line in self.lines: - line.print_points() - - - -def untar(fname): - import tarfile - tar = tarfile.open(name=fname, mode='r') - for tarinfo in tar: - tar.extract(tarinfo) - tar.close() - -def unzip(fname): - import zipfile - zf = zipfile.ZipFile(fname, 'r') - for name in zf.namelist(): - dir = os.path.dirname(name) - try: - os.makedirs(dir) - except: - pass - with open(name, 'wb') as f: - f.write(zf.read(name)) - -def read_tree(dir): - for dirpath, dirnames, filenames in os.walk(dir): - for fn in filenames: - fn = os.path.join(dirpath, fn) - if os.path.isfile(fn): - with open(fn, 'rb') as f: - f.read() - -def redirect_to_file(command, log): - return '%s > %s 2>&1' % (command, log) - -def tee_to_file(command, log): - return '%s 2>&1 | tee %s' % (command, log) - - - -class SConsTimer(object): - """ - Usage: scons-time SUBCOMMAND [ARGUMENTS] - Type "scons-time help SUBCOMMAND" for help on a specific subcommand. - - Available subcommands: - func Extract test-run data for a function - help Provides help - mem Extract --debug=memory data from test runs - obj Extract --debug=count data from test runs - time Extract --debug=time data from test runs - run Runs a test configuration - """ - - name = 'scons-time' - name_spaces = ' '*len(name) - - def makedict(**kw): - return kw - - default_settings = makedict( - chdir = None, - config_file = None, - initial_commands = [], - key_location = 'bottom left', - orig_cwd = os.getcwd(), - outdir = None, - prefix = '', - python = '"%s"' % sys.executable, - redirect = redirect_to_file, - scons = None, - scons_flags = '--debug=count --debug=memory --debug=time --debug=memoizer', - scons_lib_dir = None, - scons_wrapper = None, - startup_targets = '--help', - subdir = None, - subversion_url = None, - svn = 'svn', - svn_co_flag = '-q', - tar = 'tar', - targets = '', - targets0 = None, - targets1 = None, - targets2 = None, - title = None, - unzip = 'unzip', - verbose = False, - vertical_bars = [], - - unpack_map = { - '.tar.gz' : (untar, '%(tar)s xzf %%s'), - '.tgz' : (untar, '%(tar)s xzf %%s'), - '.tar' : (untar, '%(tar)s xf %%s'), - '.zip' : (unzip, '%(unzip)s %%s'), - }, - ) - - run_titles = [ - 'Startup', - 'Full build', - 'Up-to-date build', - ] - - run_commands = [ - '%(python)s %(scons_wrapper)s %(scons_flags)s --profile=%(prof0)s %(targets0)s', - '%(python)s %(scons_wrapper)s %(scons_flags)s --profile=%(prof1)s %(targets1)s', - '%(python)s %(scons_wrapper)s %(scons_flags)s --profile=%(prof2)s %(targets2)s', - ] - - stages = [ - 'pre-read', - 'post-read', - 'pre-build', - 'post-build', - ] - - stage_strings = { - 'pre-read' : 'Memory before reading SConscript files:', - 'post-read' : 'Memory after reading SConscript files:', - 'pre-build' : 'Memory before building targets:', - 'post-build' : 'Memory after building targets:', - } - - memory_string_all = 'Memory ' - - default_stage = stages[-1] - - time_strings = { - 'total' : 'Total build time', - 'SConscripts' : 'Total SConscript file execution time', - 'SCons' : 'Total SCons execution time', - 'commands' : 'Total command execution time', - } - - time_string_all = 'Total .* time' - - # - - def __init__(self): - self.__dict__.update(self.default_settings) - - # Functions for displaying and executing commands. - - def subst(self, x, dictionary): - try: - return x % dictionary - except TypeError: - # x isn't a string (it's probably a Python function), - # so just return it. - return x - - def subst_variables(self, command, dictionary): - """ - Substitutes (via the format operator) the values in the specified - dictionary into the specified command. - - The command can be an (action, string) tuple. In all cases, we - perform substitution on strings and don't worry if something isn't - a string. (It's probably a Python function to be executed.) - """ - try: - command + '' - except TypeError: - action = command[0] - string = command[1] - args = command[2:] - else: - action = command - string = action - args = (()) - action = self.subst(action, dictionary) - string = self.subst(string, dictionary) - return (action, string, args) - - def _do_not_display(self, msg, *args): - pass - - def display(self, msg, *args): - """ - Displays the specified message. - - Each message is prepended with a standard prefix of our name - plus the time. - """ - if callable(msg): - msg = msg(*args) - else: - msg = msg % args - if msg is None: - return - fmt = '%s[%s]: %s\n' - sys.stdout.write(fmt % (self.name, time.strftime('%H:%M:%S'), msg)) - - def _do_not_execute(self, action, *args): - pass - - def execute(self, action, *args): - """ - Executes the specified action. - - The action is called if it's a callable Python function, and - otherwise passed to os.system(). - """ - if callable(action): - action(*args) - else: - os.system(action % args) - - def run_command_list(self, commands, dict): - """ - Executes a list of commands, substituting values from the - specified dictionary. - """ - commands = [ self.subst_variables(c, dict) for c in commands ] - for action, string, args in commands: - self.display(string, *args) - sys.stdout.flush() - status = self.execute(action, *args) - if status: - sys.exit(status) - - def log_display(self, command, log): - command = self.subst(command, self.__dict__) - if log: - command = self.redirect(command, log) - return command - - def log_execute(self, command, log): - command = self.subst(command, self.__dict__) - p = os.popen(command) - output = p.read() - p.close() - #TODO: convert to subrocess, os.popen is obsolete. This didn't work: - #process = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True) - #output = process.stdout.read() - #process.stdout.close() - #process.wait() - if self.verbose: - sys.stdout.write(output) - # TODO: Figure out - # Not sure we need to write binary here - with open(log, 'w') as f: - f.write(str(output)) - - def archive_splitext(self, path): - """ - Splits an archive name into a filename base and extension. - - This is like os.path.splitext() (which it calls) except that it - also looks for '.tar.gz' and treats it as an atomic extensions. - """ - if path.endswith('.tar.gz'): - return path[:-7], path[-7:] - else: - return os.path.splitext(path) - - def args_to_files(self, args, tail=None): - """ - Takes a list of arguments, expands any glob patterns, and - returns the last "tail" files from the list. - """ - files = [] - for a in args: - files.extend(sorted(glob.glob(a))) - - if tail: - files = files[-tail:] - - return files - - def ascii_table(self, files, columns, - line_function, file_function=lambda x: x, - *args, **kw): - - header_fmt = ' '.join(['%12s'] * len(columns)) - line_fmt = header_fmt + ' %s' - - print(header_fmt % columns) - - for file in files: - t = line_function(file, *args, **kw) - if t is None: - t = [] - diff = len(columns) - len(t) - if diff > 0: - t += [''] * diff - t.append(file_function(file)) - print(line_fmt % tuple(t)) - - def collect_results(self, files, function, *args, **kw): - results = {} - - for file in files: - base = os.path.splitext(file)[0] - run, index = base.split('-')[-2:] - - run = int(run) - index = int(index) - - value = function(file, *args, **kw) - - try: - r = results[index] - except KeyError: - r = [] - results[index] = r - r.append((run, value)) - - return results - - def doc_to_help(self, obj): - """ - Translates an object's __doc__ string into help text. - - This strips a consistent number of spaces from each line in the - help text, essentially "outdenting" the text to the left-most - column. - """ - doc = obj.__doc__ - if doc is None: - return '' - return self.outdent(doc) - - def find_next_run_number(self, dir, prefix): - """ - Returns the next run number in a directory for the specified prefix. - - Examines the contents the specified directory for files with the - specified prefix, extracts the run numbers from each file name, - and returns the next run number after the largest it finds. - """ - x = re.compile(re.escape(prefix) + '-([0-9]+).*') - matches = [x.match(e) for e in os.listdir(dir)] - matches = [_f for _f in matches if _f] - if not matches: - return 0 - run_numbers = [int(m.group(1)) for m in matches] - return int(max(run_numbers)) + 1 - - def gnuplot_results(self, results, fmt='%s %.3f'): - """ - Prints out a set of results in Gnuplot format. - """ - gp = Gnuplotter(self.title, self.key_location) - - for i in sorted(results.keys()): - try: - t = self.run_titles[i] - except IndexError: - t = '??? %s ???' % i - results[i].sort() - gp.line(results[i], i+1, t, None, t, fmt=fmt) - - for bar_tuple in self.vertical_bars: - try: - x, type, label, comment = bar_tuple - except ValueError: - x, type, label = bar_tuple - comment = label - gp.vertical_bar(x, type, label, comment) - - gp.draw() - - def logfile_name(self, invocation): - """ - Returns the absolute path of a log file for the specificed - invocation number. - """ - name = self.prefix_run + '-%d.log' % invocation - return os.path.join(self.outdir, name) - - def outdent(self, s): - """ - Strip as many spaces from each line as are found at the beginning - of the first line in the list. - """ - lines = s.split('\n') - if lines[0] == '': - lines = lines[1:] - spaces = re.match(' *', lines[0]).group(0) - def strip_initial_spaces(l, s=spaces): - if l.startswith(spaces): - l = l[len(spaces):] - return l - return '\n'.join([ strip_initial_spaces(l) for l in lines ]) + '\n' - - def profile_name(self, invocation): - """ - Returns the absolute path of a profile file for the specified - invocation number. - """ - name = self.prefix_run + '-%d.prof' % invocation - return os.path.join(self.outdir, name) - - def set_env(self, key, value): - os.environ[key] = value - - # - - def get_debug_times(self, file, time_string=None): - """ - Fetch times from the --debug=time strings in the specified file. - """ - if time_string is None: - search_string = self.time_string_all - else: - search_string = time_string - with open(file) as f: - contents = f.read() - if not contents: - sys.stderr.write('file %s has no contents!\n' % repr(file)) - return None - result = re.findall(r'%s: ([\d.]*)' % search_string, contents)[-4:] - result = [ float(r) for r in result ] - if time_string is not None: - try: - result = result[0] - except IndexError: - sys.stderr.write('file %s has no results!\n' % repr(file)) - return None - return result - - def get_function_profile(self, file, function): - """ - Returns the file, line number, function name, and cumulative time. - """ - try: - import pstats - except ImportError as e: - sys.stderr.write('%s: func: %s\n' % (self.name, e)) - sys.stderr.write('%s This version of Python is missing the profiler.\n' % self.name_spaces) - sys.stderr.write('%s Cannot use the "func" subcommand.\n' % self.name_spaces) - sys.exit(1) - statistics = pstats.Stats(file).stats - matches = [ e for e in statistics.items() if e[0][2] == function ] - r = matches[0] - return r[0][0], r[0][1], r[0][2], r[1][3] - - def get_function_time(self, file, function): - """ - Returns just the cumulative time for the specified function. - """ - return self.get_function_profile(file, function)[3] - - def get_memory(self, file, memory_string=None): - """ - Returns a list of integers of the amount of memory used. The - default behavior is to return all the stages. - """ - if memory_string is None: - search_string = self.memory_string_all - else: - search_string = memory_string - with open(file) as f: - lines = f.readlines() - lines = [ l for l in lines if l.startswith(search_string) ][-4:] - result = [ int(l.split()[-1]) for l in lines[-4:] ] - if len(result) == 1: - result = result[0] - return result - - def get_object_counts(self, file, object_name, index=None): - """ - Returns the counts of the specified object_name. - """ - object_string = ' ' + object_name + '\n' - with open(file) as f: - lines = f.readlines() - line = [ l for l in lines if l.endswith(object_string) ][0] - result = [ int(field) for field in line.split()[:4] ] - if index is not None: - result = result[index] - return result - - - command_alias = {} - - def execute_subcommand(self, argv): - """ - Executes the do_*() function for the specified subcommand (argv[0]). - """ - if not argv: - return - cmdName = self.command_alias.get(argv[0], argv[0]) - try: - func = getattr(self, 'do_' + cmdName) - except AttributeError: - return self.default(argv) - try: - return func(argv) - except TypeError as e: - sys.stderr.write("%s %s: %s\n" % (self.name, cmdName, e)) - import traceback - traceback.print_exc(file=sys.stderr) - sys.stderr.write("Try '%s help %s'\n" % (self.name, cmdName)) - - def default(self, argv): - """ - The default behavior for an unknown subcommand. Prints an - error message and exits. - """ - sys.stderr.write('%s: Unknown subcommand "%s".\n' % (self.name, argv[0])) - sys.stderr.write('Type "%s help" for usage.\n' % self.name) - sys.exit(1) - - # - - def do_help(self, argv): - """ - """ - if argv[1:]: - for arg in argv[1:]: - try: - func = getattr(self, 'do_' + arg) - except AttributeError: - sys.stderr.write('%s: No help for "%s"\n' % (self.name, arg)) - else: - try: - help = getattr(self, 'help_' + arg) - except AttributeError: - sys.stdout.write(self.doc_to_help(func)) - sys.stdout.flush() - else: - help() - else: - doc = self.doc_to_help(self.__class__) - if doc: - sys.stdout.write(doc) - sys.stdout.flush() - return None - - # - - def help_func(self): - help = """\ - Usage: scons-time func [OPTIONS] FILE [...] - - -C DIR, --chdir=DIR Change to DIR before looking for files - -f FILE, --file=FILE Read configuration from specified FILE - --fmt=FORMAT, --format=FORMAT Print data in specified FORMAT - --func=NAME, --function=NAME Report time for function NAME - -h, --help Print this help and exit - -p STRING, --prefix=STRING Use STRING as log file/profile prefix - -t NUMBER, --tail=NUMBER Only report the last NUMBER files - --title=TITLE Specify the output plot TITLE - """ - sys.stdout.write(self.outdent(help)) - sys.stdout.flush() - - def do_func(self, argv): - """ - """ - format = 'ascii' - function_name = '_main' - tail = None - - short_opts = '?C:f:hp:t:' - - long_opts = [ - 'chdir=', - 'file=', - 'fmt=', - 'format=', - 'func=', - 'function=', - 'help', - 'prefix=', - 'tail=', - 'title=', - ] - - opts, args = getopt.getopt(argv[1:], short_opts, long_opts) - - for o, a in opts: - if o in ('-C', '--chdir'): - self.chdir = a - elif o in ('-f', '--file'): - self.config_file = a - elif o in ('--fmt', '--format'): - format = a - elif o in ('--func', '--function'): - function_name = a - elif o in ('-?', '-h', '--help'): - self.do_help(['help', 'func']) - sys.exit(0) - elif o in ('--max',): - max_time = int(a) - elif o in ('-p', '--prefix'): - self.prefix = a - elif o in ('-t', '--tail'): - tail = int(a) - elif o in ('--title',): - self.title = a - - if self.config_file: - with open(self.config_file, 'r') as f: - config = f.read() - exec(config, self.__dict__) - - if self.chdir: - os.chdir(self.chdir) - - if not args: - - pattern = '%s*.prof' % self.prefix - args = self.args_to_files([pattern], tail) - - if not args: - if self.chdir: - directory = self.chdir - else: - directory = os.getcwd() - - sys.stderr.write('%s: func: No arguments specified.\n' % self.name) - sys.stderr.write('%s No %s*.prof files found in "%s".\n' % (self.name_spaces, self.prefix, directory)) - sys.stderr.write('%s Type "%s help func" for help.\n' % (self.name_spaces, self.name)) - sys.exit(1) - - else: - - args = self.args_to_files(args, tail) - - cwd_ = os.getcwd() + os.sep - - if format == 'ascii': - - for file in args: - try: - f, line, func, time = \ - self.get_function_profile(file, function_name) - except ValueError as e: - sys.stderr.write("%s: func: %s: %s\n" % - (self.name, file, e)) - else: - if f.startswith(cwd_): - f = f[len(cwd_):] - print("%.3f %s:%d(%s)" % (time, f, line, func)) - - elif format == 'gnuplot': - - results = self.collect_results(args, self.get_function_time, - function_name) - - self.gnuplot_results(results) - - else: - - sys.stderr.write('%s: func: Unknown format "%s".\n' % (self.name, format)) - sys.exit(1) - - # - - def help_mem(self): - help = """\ - Usage: scons-time mem [OPTIONS] FILE [...] - - -C DIR, --chdir=DIR Change to DIR before looking for files - -f FILE, --file=FILE Read configuration from specified FILE - --fmt=FORMAT, --format=FORMAT Print data in specified FORMAT - -h, --help Print this help and exit - -p STRING, --prefix=STRING Use STRING as log file/profile prefix - --stage=STAGE Plot memory at the specified stage: - pre-read, post-read, pre-build, - post-build (default: post-build) - -t NUMBER, --tail=NUMBER Only report the last NUMBER files - --title=TITLE Specify the output plot TITLE - """ - sys.stdout.write(self.outdent(help)) - sys.stdout.flush() - - def do_mem(self, argv): - - format = 'ascii' - logfile_path = lambda x: x - stage = self.default_stage - tail = None - - short_opts = '?C:f:hp:t:' - - long_opts = [ - 'chdir=', - 'file=', - 'fmt=', - 'format=', - 'help', - 'prefix=', - 'stage=', - 'tail=', - 'title=', - ] - - opts, args = getopt.getopt(argv[1:], short_opts, long_opts) - - for o, a in opts: - if o in ('-C', '--chdir'): - self.chdir = a - elif o in ('-f', '--file'): - self.config_file = a - elif o in ('--fmt', '--format'): - format = a - elif o in ('-?', '-h', '--help'): - self.do_help(['help', 'mem']) - sys.exit(0) - elif o in ('-p', '--prefix'): - self.prefix = a - elif o in ('--stage',): - if a not in self.stages: - sys.stderr.write('%s: mem: Unrecognized stage "%s".\n' % (self.name, a)) - sys.exit(1) - stage = a - elif o in ('-t', '--tail'): - tail = int(a) - elif o in ('--title',): - self.title = a - - if self.config_file: - with open(self.config_file, 'r') as f: - config = f.read() - HACK_for_exec(config, self.__dict__) - - if self.chdir: - os.chdir(self.chdir) - logfile_path = lambda x: os.path.join(self.chdir, x) - - if not args: - - pattern = '%s*.log' % self.prefix - args = self.args_to_files([pattern], tail) - - if not args: - if self.chdir: - directory = self.chdir - else: - directory = os.getcwd() - - sys.stderr.write('%s: mem: No arguments specified.\n' % self.name) - sys.stderr.write('%s No %s*.log files found in "%s".\n' % (self.name_spaces, self.prefix, directory)) - sys.stderr.write('%s Type "%s help mem" for help.\n' % (self.name_spaces, self.name)) - sys.exit(1) - - else: - - args = self.args_to_files(args, tail) - - cwd_ = os.getcwd() + os.sep - - if format == 'ascii': - - self.ascii_table(args, tuple(self.stages), self.get_memory, logfile_path) - - elif format == 'gnuplot': - - results = self.collect_results(args, self.get_memory, - self.stage_strings[stage]) - - self.gnuplot_results(results) - - else: - - sys.stderr.write('%s: mem: Unknown format "%s".\n' % (self.name, format)) - sys.exit(1) - - return 0 - - # - - def help_obj(self): - help = """\ - Usage: scons-time obj [OPTIONS] OBJECT FILE [...] - - -C DIR, --chdir=DIR Change to DIR before looking for files - -f FILE, --file=FILE Read configuration from specified FILE - --fmt=FORMAT, --format=FORMAT Print data in specified FORMAT - -h, --help Print this help and exit - -p STRING, --prefix=STRING Use STRING as log file/profile prefix - --stage=STAGE Plot memory at the specified stage: - pre-read, post-read, pre-build, - post-build (default: post-build) - -t NUMBER, --tail=NUMBER Only report the last NUMBER files - --title=TITLE Specify the output plot TITLE - """ - sys.stdout.write(self.outdent(help)) - sys.stdout.flush() - - def do_obj(self, argv): - - format = 'ascii' - logfile_path = lambda x: x - stage = self.default_stage - tail = None - - short_opts = '?C:f:hp:t:' - - long_opts = [ - 'chdir=', - 'file=', - 'fmt=', - 'format=', - 'help', - 'prefix=', - 'stage=', - 'tail=', - 'title=', - ] - - opts, args = getopt.getopt(argv[1:], short_opts, long_opts) - - for o, a in opts: - if o in ('-C', '--chdir'): - self.chdir = a - elif o in ('-f', '--file'): - self.config_file = a - elif o in ('--fmt', '--format'): - format = a - elif o in ('-?', '-h', '--help'): - self.do_help(['help', 'obj']) - sys.exit(0) - elif o in ('-p', '--prefix'): - self.prefix = a - elif o in ('--stage',): - if a not in self.stages: - sys.stderr.write('%s: obj: Unrecognized stage "%s".\n' % (self.name, a)) - sys.stderr.write('%s Type "%s help obj" for help.\n' % (self.name_spaces, self.name)) - sys.exit(1) - stage = a - elif o in ('-t', '--tail'): - tail = int(a) - elif o in ('--title',): - self.title = a - - if not args: - sys.stderr.write('%s: obj: Must specify an object name.\n' % self.name) - sys.stderr.write('%s Type "%s help obj" for help.\n' % (self.name_spaces, self.name)) - sys.exit(1) - - object_name = args.pop(0) - - if self.config_file: - with open(self.config_file, 'r') as f: - config = f.read() - HACK_for_exec(config, self.__dict__) - - if self.chdir: - os.chdir(self.chdir) - logfile_path = lambda x: os.path.join(self.chdir, x) - - if not args: - - pattern = '%s*.log' % self.prefix - args = self.args_to_files([pattern], tail) - - if not args: - if self.chdir: - directory = self.chdir - else: - directory = os.getcwd() - - sys.stderr.write('%s: obj: No arguments specified.\n' % self.name) - sys.stderr.write('%s No %s*.log files found in "%s".\n' % (self.name_spaces, self.prefix, directory)) - sys.stderr.write('%s Type "%s help obj" for help.\n' % (self.name_spaces, self.name)) - sys.exit(1) - - else: - - args = self.args_to_files(args, tail) - - cwd_ = os.getcwd() + os.sep - - if format == 'ascii': - - self.ascii_table(args, tuple(self.stages), self.get_object_counts, logfile_path, object_name) - - elif format == 'gnuplot': - - stage_index = 0 - for s in self.stages: - if stage == s: - break - stage_index = stage_index + 1 - - results = self.collect_results(args, self.get_object_counts, - object_name, stage_index) - - self.gnuplot_results(results) - - else: - - sys.stderr.write('%s: obj: Unknown format "%s".\n' % (self.name, format)) - sys.exit(1) - - return 0 - - # - - def help_run(self): - help = """\ - Usage: scons-time run [OPTIONS] [FILE ...] - - --chdir=DIR Name of unpacked directory for chdir - -f FILE, --file=FILE Read configuration from specified FILE - -h, --help Print this help and exit - -n, --no-exec No execute, just print command lines - --number=NUMBER Put output in files for run NUMBER - --outdir=OUTDIR Put output files in OUTDIR - -p STRING, --prefix=STRING Use STRING as log file/profile prefix - --python=PYTHON Time using the specified PYTHON - -q, --quiet Don't print command lines - --scons=SCONS Time using the specified SCONS - --svn=URL, --subversion=URL Use SCons from Subversion URL - -v, --verbose Display output of commands - """ - sys.stdout.write(self.outdent(help)) - sys.stdout.flush() - - def do_run(self, argv): - """ - """ - run_number_list = [None] - - short_opts = '?f:hnp:qs:v' - - long_opts = [ - 'file=', - 'help', - 'no-exec', - 'number=', - 'outdir=', - 'prefix=', - 'python=', - 'quiet', - 'scons=', - 'svn=', - 'subdir=', - 'subversion=', - 'verbose', - ] - - opts, args = getopt.getopt(argv[1:], short_opts, long_opts) - - for o, a in opts: - if o in ('-f', '--file'): - self.config_file = a - elif o in ('-?', '-h', '--help'): - self.do_help(['help', 'run']) - sys.exit(0) - elif o in ('-n', '--no-exec'): - self.execute = self._do_not_execute - elif o in ('--number',): - run_number_list = self.split_run_numbers(a) - elif o in ('--outdir',): - self.outdir = a - elif o in ('-p', '--prefix'): - self.prefix = a - elif o in ('--python',): - self.python = a - elif o in ('-q', '--quiet'): - self.display = self._do_not_display - elif o in ('-s', '--subdir'): - self.subdir = a - elif o in ('--scons',): - self.scons = a - elif o in ('--svn', '--subversion'): - self.subversion_url = a - elif o in ('-v', '--verbose'): - self.redirect = tee_to_file - self.verbose = True - self.svn_co_flag = '' - - if not args and not self.config_file: - sys.stderr.write('%s: run: No arguments or -f config file specified.\n' % self.name) - sys.stderr.write('%s Type "%s help run" for help.\n' % (self.name_spaces, self.name)) - sys.exit(1) - - if self.config_file: - with open(self.config_file, 'r') as f: - config = f.read() - exec(config, self.__dict__) - - if args: - self.archive_list = args - - archive_file_name = os.path.split(self.archive_list[0])[1] - - if not self.subdir: - self.subdir = self.archive_splitext(archive_file_name)[0] - - if not self.prefix: - self.prefix = self.archive_splitext(archive_file_name)[0] - - prepare = None - if self.subversion_url: - prepare = self.prep_subversion_run - - for run_number in run_number_list: - self.individual_run(run_number, self.archive_list, prepare) - - def split_run_numbers(self, s): - result = [] - for n in s.split(','): - try: - x, y = n.split('-') - except ValueError: - result.append(int(n)) - else: - result.extend(list(range(int(x), int(y)+1))) - return result - - def scons_path(self, dir): - return os.path.join(dir, 'src', 'script', 'scons.py') - - def scons_lib_dir_path(self, dir): - return os.path.join(dir, 'src', 'engine') - - def prep_subversion_run(self, commands, removals): - self.svn_tmpdir = tempfile.mkdtemp(prefix=self.name + '-svn-') - removals.append((shutil.rmtree, 'rm -rf %%s', self.svn_tmpdir)) - - self.scons = self.scons_path(self.svn_tmpdir) - self.scons_lib_dir = self.scons_lib_dir_path(self.svn_tmpdir) - - commands.extend([ - '%(svn)s co %(svn_co_flag)s -r %(run_number)s %(subversion_url)s %(svn_tmpdir)s', - ]) - - def individual_run(self, run_number, archive_list, prepare=None): - """ - Performs an individual run of the default SCons invocations. - """ - - commands = [] - removals = [] - - if prepare: - prepare(commands, removals) - - save_scons = self.scons - save_scons_wrapper = self.scons_wrapper - save_scons_lib_dir = self.scons_lib_dir - - if self.outdir is None: - self.outdir = self.orig_cwd - elif not os.path.isabs(self.outdir): - self.outdir = os.path.join(self.orig_cwd, self.outdir) - - if self.scons is None: - self.scons = self.scons_path(self.orig_cwd) - - if self.scons_lib_dir is None: - self.scons_lib_dir = self.scons_lib_dir_path(self.orig_cwd) - - if self.scons_wrapper is None: - self.scons_wrapper = self.scons - - if not run_number: - run_number = self.find_next_run_number(self.outdir, self.prefix) - - self.run_number = str(run_number) - - self.prefix_run = self.prefix + '-%03d' % run_number - - if self.targets0 is None: - self.targets0 = self.startup_targets - if self.targets1 is None: - self.targets1 = self.targets - if self.targets2 is None: - self.targets2 = self.targets - - self.tmpdir = tempfile.mkdtemp(prefix=self.name + '-') - - commands.extend([ - (os.chdir, 'cd %%s', self.tmpdir), - ]) - - for archive in archive_list: - if not os.path.isabs(archive): - archive = os.path.join(self.orig_cwd, archive) - if os.path.isdir(archive): - dest = os.path.split(archive)[1] - commands.append((shutil.copytree, 'cp -r %%s %%s', archive, dest)) - else: - suffix = self.archive_splitext(archive)[1] - unpack_command = self.unpack_map.get(suffix) - if not unpack_command: - dest = os.path.split(archive)[1] - commands.append((shutil.copyfile, 'cp %%s %%s', archive, dest)) - else: - commands.append(unpack_command + (archive,)) - - commands.extend([ - (os.chdir, 'cd %%s', self.subdir), - ]) - - commands.extend(self.initial_commands) - - commands.extend([ - (lambda: read_tree('.'), - 'find * -type f | xargs cat > /dev/null'), - - (self.set_env, 'export %%s=%%s', - 'SCONS_LIB_DIR', self.scons_lib_dir), - - '%(python)s %(scons_wrapper)s --version', - ]) - - index = 0 - for run_command in self.run_commands: - setattr(self, 'prof%d' % index, self.profile_name(index)) - c = ( - self.log_execute, - self.log_display, - run_command, - self.logfile_name(index), - ) - commands.append(c) - index = index + 1 - - commands.extend([ - (os.chdir, 'cd %%s', self.orig_cwd), - ]) - - if not os.environ.get('PRESERVE'): - commands.extend(removals) - commands.append((shutil.rmtree, 'rm -rf %%s', self.tmpdir)) - - self.run_command_list(commands, self.__dict__) - - self.scons = save_scons - self.scons_lib_dir = save_scons_lib_dir - self.scons_wrapper = save_scons_wrapper - - # - - def help_time(self): - help = """\ - Usage: scons-time time [OPTIONS] FILE [...] - - -C DIR, --chdir=DIR Change to DIR before looking for files - -f FILE, --file=FILE Read configuration from specified FILE - --fmt=FORMAT, --format=FORMAT Print data in specified FORMAT - -h, --help Print this help and exit - -p STRING, --prefix=STRING Use STRING as log file/profile prefix - -t NUMBER, --tail=NUMBER Only report the last NUMBER files - --which=TIMER Plot timings for TIMER: total, - SConscripts, SCons, commands. - """ - sys.stdout.write(self.outdent(help)) - sys.stdout.flush() - - def do_time(self, argv): - - format = 'ascii' - logfile_path = lambda x: x - tail = None - which = 'total' - - short_opts = '?C:f:hp:t:' - - long_opts = [ - 'chdir=', - 'file=', - 'fmt=', - 'format=', - 'help', - 'prefix=', - 'tail=', - 'title=', - 'which=', - ] - - opts, args = getopt.getopt(argv[1:], short_opts, long_opts) - - for o, a in opts: - if o in ('-C', '--chdir'): - self.chdir = a - elif o in ('-f', '--file'): - self.config_file = a - elif o in ('--fmt', '--format'): - format = a - elif o in ('-?', '-h', '--help'): - self.do_help(['help', 'time']) - sys.exit(0) - elif o in ('-p', '--prefix'): - self.prefix = a - elif o in ('-t', '--tail'): - tail = int(a) - elif o in ('--title',): - self.title = a - elif o in ('--which',): - if a not in list(self.time_strings.keys()): - sys.stderr.write('%s: time: Unrecognized timer "%s".\n' % (self.name, a)) - sys.stderr.write('%s Type "%s help time" for help.\n' % (self.name_spaces, self.name)) - sys.exit(1) - which = a - - if self.config_file: - with open(self.config_file, 'r') as f: - config = f.read() - HACK_for_exec(config, self.__dict__) - - if self.chdir: - os.chdir(self.chdir) - logfile_path = lambda x: os.path.join(self.chdir, x) - - if not args: - - pattern = '%s*.log' % self.prefix - args = self.args_to_files([pattern], tail) - - if not args: - if self.chdir: - directory = self.chdir - else: - directory = os.getcwd() - - sys.stderr.write('%s: time: No arguments specified.\n' % self.name) - sys.stderr.write('%s No %s*.log files found in "%s".\n' % (self.name_spaces, self.prefix, directory)) - sys.stderr.write('%s Type "%s help time" for help.\n' % (self.name_spaces, self.name)) - sys.exit(1) - - else: - - args = self.args_to_files(args, tail) - - cwd_ = os.getcwd() + os.sep - - if format == 'ascii': - - columns = ("Total", "SConscripts", "SCons", "commands") - self.ascii_table(args, columns, self.get_debug_times, logfile_path) - - elif format == 'gnuplot': - - results = self.collect_results(args, self.get_debug_times, - self.time_strings[which]) - - self.gnuplot_results(results, fmt='%s %.6f') - - else: - - sys.stderr.write('%s: time: Unknown format "%s".\n' % (self.name, format)) - sys.exit(1) - -if __name__ == '__main__': - opts, args = getopt.getopt(sys.argv[1:], 'h?V', ['help', 'version']) - - ST = SConsTimer() - - for o, a in opts: - if o in ('-?', '-h', '--help'): - ST.do_help(['help']) - sys.exit(0) - elif o in ('-V', '--version'): - sys.stdout.write('scons-time version\n') - sys.exit(0) - - if not args: - sys.stderr.write('Type "%s help" for usage.\n' % ST.name) - sys.exit(1) - - ST.execute_subcommand(args) - -# Local Variables: -# tab-width:4 -# indent-tabs-mode:nil -# End: -# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/scripts/scons-time.py b/scripts/scons-time.py new file mode 100644 index 0000000..6494349 --- /dev/null +++ b/scripts/scons-time.py @@ -0,0 +1,1480 @@ +#!/usr/bin/env python +# +# scons-time - run SCons timings and collect statistics +# +# A script for running a configuration through SCons with a standard +# set of invocations to collect timing and memory statistics and to +# capture the results in a consistent set of output files for display +# and analysis. +# + +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +import getopt +import glob +import os +import re +import shutil +import sys +import tempfile +import time +import subprocess + +def HACK_for_exec(cmd, *args): + """ + For some reason, Python won't allow an exec() within a function + that also declares an internal function (including lambda functions). + This function is a hack that calls exec() in a function with no + internal functions. + """ + if not args: exec(cmd) + elif len(args) == 1: exec(cmd, args[0]) + else: exec(cmd, args[0], args[1]) + +class Plotter(object): + def increment_size(self, largest): + """ + Return the size of each horizontal increment line for a specified + maximum value. This returns a value that will provide somewhere + between 5 and 9 horizontal lines on the graph, on some set of + boundaries that are multiples of 10/100/1000/etc. + """ + i = largest // 5 + if not i: + return largest + multiplier = 1 + while i >= 10: + i = i // 10 + multiplier = multiplier * 10 + return i * multiplier + + def max_graph_value(self, largest): + # Round up to next integer. + largest = int(largest) + 1 + increment = self.increment_size(largest) + return ((largest + increment - 1) // increment) * increment + +class Line(object): + def __init__(self, points, type, title, label, comment, fmt="%s %s"): + self.points = points + self.type = type + self.title = title + self.label = label + self.comment = comment + self.fmt = fmt + + def print_label(self, inx, x, y): + if self.label: + print('set label %s "%s" at %0.1f,%0.1f right' % (inx, self.label, x, y)) + + def plot_string(self): + if self.title: + title_string = 'title "%s"' % self.title + else: + title_string = 'notitle' + return "'-' %s with lines lt %s" % (title_string, self.type) + + def print_points(self, fmt=None): + if fmt is None: + fmt = self.fmt + if self.comment: + print('# %s' % self.comment) + for x, y in self.points: + # If y is None, it usually represents some kind of break + # in the line's index number. We might want to represent + # this some way rather than just drawing the line straight + # between the two points on either side. + if y is not None: + print(fmt % (x, y)) + print('e') + + def get_x_values(self): + return [ p[0] for p in self.points ] + + def get_y_values(self): + return [ p[1] for p in self.points ] + +class Gnuplotter(Plotter): + + def __init__(self, title, key_location): + self.lines = [] + self.title = title + self.key_location = key_location + + def line(self, points, type, title=None, label=None, comment=None, fmt='%s %s'): + if points: + line = Line(points, type, title, label, comment, fmt) + self.lines.append(line) + + def plot_string(self, line): + return line.plot_string() + + def vertical_bar(self, x, type, label, comment): + if self.get_min_x() <= x <= self.get_max_x(): + points = [(x, 0), (x, self.max_graph_value(self.get_max_y()))] + self.line(points, type, label, comment) + + def get_all_x_values(self): + result = [] + for line in self.lines: + result.extend(line.get_x_values()) + return [r for r in result if r is not None] + + def get_all_y_values(self): + result = [] + for line in self.lines: + result.extend(line.get_y_values()) + return [r for r in result if r is not None] + + def get_min_x(self): + try: + return self.min_x + except AttributeError: + try: + self.min_x = min(self.get_all_x_values()) + except ValueError: + self.min_x = 0 + return self.min_x + + def get_max_x(self): + try: + return self.max_x + except AttributeError: + try: + self.max_x = max(self.get_all_x_values()) + except ValueError: + self.max_x = 0 + return self.max_x + + def get_min_y(self): + try: + return self.min_y + except AttributeError: + try: + self.min_y = min(self.get_all_y_values()) + except ValueError: + self.min_y = 0 + return self.min_y + + def get_max_y(self): + try: + return self.max_y + except AttributeError: + try: + self.max_y = max(self.get_all_y_values()) + except ValueError: + self.max_y = 0 + return self.max_y + + def draw(self): + + if not self.lines: + return + + if self.title: + print('set title "%s"' % self.title) + print('set key %s' % self.key_location) + + min_y = self.get_min_y() + max_y = self.max_graph_value(self.get_max_y()) + incr = (max_y - min_y) / 10.0 + start = min_y + (max_y / 2.0) + (2.0 * incr) + position = [ start - (i * incr) for i in range(5) ] + + inx = 1 + for line in self.lines: + line.print_label(inx, line.points[0][0]-1, + position[(inx-1) % len(position)]) + inx += 1 + + plot_strings = [ self.plot_string(l) for l in self.lines ] + print('plot ' + ', \\\n '.join(plot_strings)) + + for line in self.lines: + line.print_points() + + + +def untar(fname): + import tarfile + tar = tarfile.open(name=fname, mode='r') + for tarinfo in tar: + tar.extract(tarinfo) + tar.close() + +def unzip(fname): + import zipfile + zf = zipfile.ZipFile(fname, 'r') + for name in zf.namelist(): + dir = os.path.dirname(name) + try: + os.makedirs(dir) + except: + pass + with open(name, 'wb') as f: + f.write(zf.read(name)) + +def read_tree(dir): + for dirpath, dirnames, filenames in os.walk(dir): + for fn in filenames: + fn = os.path.join(dirpath, fn) + if os.path.isfile(fn): + with open(fn, 'rb') as f: + f.read() + +def redirect_to_file(command, log): + return '%s > %s 2>&1' % (command, log) + +def tee_to_file(command, log): + return '%s 2>&1 | tee %s' % (command, log) + + + +class SConsTimer(object): + """ + Usage: scons-time SUBCOMMAND [ARGUMENTS] + Type "scons-time help SUBCOMMAND" for help on a specific subcommand. + + Available subcommands: + func Extract test-run data for a function + help Provides help + mem Extract --debug=memory data from test runs + obj Extract --debug=count data from test runs + time Extract --debug=time data from test runs + run Runs a test configuration + """ + + name = 'scons-time' + name_spaces = ' '*len(name) + + def makedict(**kw): + return kw + + default_settings = makedict( + chdir = None, + config_file = None, + initial_commands = [], + key_location = 'bottom left', + orig_cwd = os.getcwd(), + outdir = None, + prefix = '', + python = '"%s"' % sys.executable, + redirect = redirect_to_file, + scons = None, + scons_flags = '--debug=count --debug=memory --debug=time --debug=memoizer', + scons_lib_dir = None, + scons_wrapper = None, + startup_targets = '--help', + subdir = None, + subversion_url = None, + svn = 'svn', + svn_co_flag = '-q', + tar = 'tar', + targets = '', + targets0 = None, + targets1 = None, + targets2 = None, + title = None, + unzip = 'unzip', + verbose = False, + vertical_bars = [], + + unpack_map = { + '.tar.gz' : (untar, '%(tar)s xzf %%s'), + '.tgz' : (untar, '%(tar)s xzf %%s'), + '.tar' : (untar, '%(tar)s xf %%s'), + '.zip' : (unzip, '%(unzip)s %%s'), + }, + ) + + run_titles = [ + 'Startup', + 'Full build', + 'Up-to-date build', + ] + + run_commands = [ + '%(python)s %(scons_wrapper)s %(scons_flags)s --profile=%(prof0)s %(targets0)s', + '%(python)s %(scons_wrapper)s %(scons_flags)s --profile=%(prof1)s %(targets1)s', + '%(python)s %(scons_wrapper)s %(scons_flags)s --profile=%(prof2)s %(targets2)s', + ] + + stages = [ + 'pre-read', + 'post-read', + 'pre-build', + 'post-build', + ] + + stage_strings = { + 'pre-read' : 'Memory before reading SConscript files:', + 'post-read' : 'Memory after reading SConscript files:', + 'pre-build' : 'Memory before building targets:', + 'post-build' : 'Memory after building targets:', + } + + memory_string_all = 'Memory ' + + default_stage = stages[-1] + + time_strings = { + 'total' : 'Total build time', + 'SConscripts' : 'Total SConscript file execution time', + 'SCons' : 'Total SCons execution time', + 'commands' : 'Total command execution time', + } + + time_string_all = 'Total .* time' + + # + + def __init__(self): + self.__dict__.update(self.default_settings) + + # Functions for displaying and executing commands. + + def subst(self, x, dictionary): + try: + return x % dictionary + except TypeError: + # x isn't a string (it's probably a Python function), + # so just return it. + return x + + def subst_variables(self, command, dictionary): + """ + Substitutes (via the format operator) the values in the specified + dictionary into the specified command. + + The command can be an (action, string) tuple. In all cases, we + perform substitution on strings and don't worry if something isn't + a string. (It's probably a Python function to be executed.) + """ + try: + command + '' + except TypeError: + action = command[0] + string = command[1] + args = command[2:] + else: + action = command + string = action + args = (()) + action = self.subst(action, dictionary) + string = self.subst(string, dictionary) + return (action, string, args) + + def _do_not_display(self, msg, *args): + pass + + def display(self, msg, *args): + """ + Displays the specified message. + + Each message is prepended with a standard prefix of our name + plus the time. + """ + if callable(msg): + msg = msg(*args) + else: + msg = msg % args + if msg is None: + return + fmt = '%s[%s]: %s\n' + sys.stdout.write(fmt % (self.name, time.strftime('%H:%M:%S'), msg)) + + def _do_not_execute(self, action, *args): + pass + + def execute(self, action, *args): + """ + Executes the specified action. + + The action is called if it's a callable Python function, and + otherwise passed to os.system(). + """ + if callable(action): + action(*args) + else: + os.system(action % args) + + def run_command_list(self, commands, dict): + """ + Executes a list of commands, substituting values from the + specified dictionary. + """ + commands = [ self.subst_variables(c, dict) for c in commands ] + for action, string, args in commands: + self.display(string, *args) + sys.stdout.flush() + status = self.execute(action, *args) + if status: + sys.exit(status) + + def log_display(self, command, log): + command = self.subst(command, self.__dict__) + if log: + command = self.redirect(command, log) + return command + + def log_execute(self, command, log): + command = self.subst(command, self.__dict__) + p = os.popen(command) + output = p.read() + p.close() + #TODO: convert to subrocess, os.popen is obsolete. This didn't work: + #process = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True) + #output = process.stdout.read() + #process.stdout.close() + #process.wait() + if self.verbose: + sys.stdout.write(output) + # TODO: Figure out + # Not sure we need to write binary here + with open(log, 'w') as f: + f.write(str(output)) + + def archive_splitext(self, path): + """ + Splits an archive name into a filename base and extension. + + This is like os.path.splitext() (which it calls) except that it + also looks for '.tar.gz' and treats it as an atomic extensions. + """ + if path.endswith('.tar.gz'): + return path[:-7], path[-7:] + else: + return os.path.splitext(path) + + def args_to_files(self, args, tail=None): + """ + Takes a list of arguments, expands any glob patterns, and + returns the last "tail" files from the list. + """ + files = [] + for a in args: + files.extend(sorted(glob.glob(a))) + + if tail: + files = files[-tail:] + + return files + + def ascii_table(self, files, columns, + line_function, file_function=lambda x: x, + *args, **kw): + + header_fmt = ' '.join(['%12s'] * len(columns)) + line_fmt = header_fmt + ' %s' + + print(header_fmt % columns) + + for file in files: + t = line_function(file, *args, **kw) + if t is None: + t = [] + diff = len(columns) - len(t) + if diff > 0: + t += [''] * diff + t.append(file_function(file)) + print(line_fmt % tuple(t)) + + def collect_results(self, files, function, *args, **kw): + results = {} + + for file in files: + base = os.path.splitext(file)[0] + run, index = base.split('-')[-2:] + + run = int(run) + index = int(index) + + value = function(file, *args, **kw) + + try: + r = results[index] + except KeyError: + r = [] + results[index] = r + r.append((run, value)) + + return results + + def doc_to_help(self, obj): + """ + Translates an object's __doc__ string into help text. + + This strips a consistent number of spaces from each line in the + help text, essentially "outdenting" the text to the left-most + column. + """ + doc = obj.__doc__ + if doc is None: + return '' + return self.outdent(doc) + + def find_next_run_number(self, dir, prefix): + """ + Returns the next run number in a directory for the specified prefix. + + Examines the contents the specified directory for files with the + specified prefix, extracts the run numbers from each file name, + and returns the next run number after the largest it finds. + """ + x = re.compile(re.escape(prefix) + '-([0-9]+).*') + matches = [x.match(e) for e in os.listdir(dir)] + matches = [_f for _f in matches if _f] + if not matches: + return 0 + run_numbers = [int(m.group(1)) for m in matches] + return int(max(run_numbers)) + 1 + + def gnuplot_results(self, results, fmt='%s %.3f'): + """ + Prints out a set of results in Gnuplot format. + """ + gp = Gnuplotter(self.title, self.key_location) + + for i in sorted(results.keys()): + try: + t = self.run_titles[i] + except IndexError: + t = '??? %s ???' % i + results[i].sort() + gp.line(results[i], i+1, t, None, t, fmt=fmt) + + for bar_tuple in self.vertical_bars: + try: + x, type, label, comment = bar_tuple + except ValueError: + x, type, label = bar_tuple + comment = label + gp.vertical_bar(x, type, label, comment) + + gp.draw() + + def logfile_name(self, invocation): + """ + Returns the absolute path of a log file for the specificed + invocation number. + """ + name = self.prefix_run + '-%d.log' % invocation + return os.path.join(self.outdir, name) + + def outdent(self, s): + """ + Strip as many spaces from each line as are found at the beginning + of the first line in the list. + """ + lines = s.split('\n') + if lines[0] == '': + lines = lines[1:] + spaces = re.match(' *', lines[0]).group(0) + def strip_initial_spaces(l, s=spaces): + if l.startswith(spaces): + l = l[len(spaces):] + return l + return '\n'.join([ strip_initial_spaces(l) for l in lines ]) + '\n' + + def profile_name(self, invocation): + """ + Returns the absolute path of a profile file for the specified + invocation number. + """ + name = self.prefix_run + '-%d.prof' % invocation + return os.path.join(self.outdir, name) + + def set_env(self, key, value): + os.environ[key] = value + + # + + def get_debug_times(self, file, time_string=None): + """ + Fetch times from the --debug=time strings in the specified file. + """ + if time_string is None: + search_string = self.time_string_all + else: + search_string = time_string + with open(file) as f: + contents = f.read() + if not contents: + sys.stderr.write('file %s has no contents!\n' % repr(file)) + return None + result = re.findall(r'%s: ([\d.]*)' % search_string, contents)[-4:] + result = [ float(r) for r in result ] + if time_string is not None: + try: + result = result[0] + except IndexError: + sys.stderr.write('file %s has no results!\n' % repr(file)) + return None + return result + + def get_function_profile(self, file, function): + """ + Returns the file, line number, function name, and cumulative time. + """ + try: + import pstats + except ImportError as e: + sys.stderr.write('%s: func: %s\n' % (self.name, e)) + sys.stderr.write('%s This version of Python is missing the profiler.\n' % self.name_spaces) + sys.stderr.write('%s Cannot use the "func" subcommand.\n' % self.name_spaces) + sys.exit(1) + statistics = pstats.Stats(file).stats + matches = [ e for e in statistics.items() if e[0][2] == function ] + r = matches[0] + return r[0][0], r[0][1], r[0][2], r[1][3] + + def get_function_time(self, file, function): + """ + Returns just the cumulative time for the specified function. + """ + return self.get_function_profile(file, function)[3] + + def get_memory(self, file, memory_string=None): + """ + Returns a list of integers of the amount of memory used. The + default behavior is to return all the stages. + """ + if memory_string is None: + search_string = self.memory_string_all + else: + search_string = memory_string + with open(file) as f: + lines = f.readlines() + lines = [ l for l in lines if l.startswith(search_string) ][-4:] + result = [ int(l.split()[-1]) for l in lines[-4:] ] + if len(result) == 1: + result = result[0] + return result + + def get_object_counts(self, file, object_name, index=None): + """ + Returns the counts of the specified object_name. + """ + object_string = ' ' + object_name + '\n' + with open(file) as f: + lines = f.readlines() + line = [ l for l in lines if l.endswith(object_string) ][0] + result = [ int(field) for field in line.split()[:4] ] + if index is not None: + result = result[index] + return result + + + command_alias = {} + + def execute_subcommand(self, argv): + """ + Executes the do_*() function for the specified subcommand (argv[0]). + """ + if not argv: + return + cmdName = self.command_alias.get(argv[0], argv[0]) + try: + func = getattr(self, 'do_' + cmdName) + except AttributeError: + return self.default(argv) + try: + return func(argv) + except TypeError as e: + sys.stderr.write("%s %s: %s\n" % (self.name, cmdName, e)) + import traceback + traceback.print_exc(file=sys.stderr) + sys.stderr.write("Try '%s help %s'\n" % (self.name, cmdName)) + + def default(self, argv): + """ + The default behavior for an unknown subcommand. Prints an + error message and exits. + """ + sys.stderr.write('%s: Unknown subcommand "%s".\n' % (self.name, argv[0])) + sys.stderr.write('Type "%s help" for usage.\n' % self.name) + sys.exit(1) + + # + + def do_help(self, argv): + """ + """ + if argv[1:]: + for arg in argv[1:]: + try: + func = getattr(self, 'do_' + arg) + except AttributeError: + sys.stderr.write('%s: No help for "%s"\n' % (self.name, arg)) + else: + try: + help = getattr(self, 'help_' + arg) + except AttributeError: + sys.stdout.write(self.doc_to_help(func)) + sys.stdout.flush() + else: + help() + else: + doc = self.doc_to_help(self.__class__) + if doc: + sys.stdout.write(doc) + sys.stdout.flush() + return None + + # + + def help_func(self): + help = """\ + Usage: scons-time func [OPTIONS] FILE [...] + + -C DIR, --chdir=DIR Change to DIR before looking for files + -f FILE, --file=FILE Read configuration from specified FILE + --fmt=FORMAT, --format=FORMAT Print data in specified FORMAT + --func=NAME, --function=NAME Report time for function NAME + -h, --help Print this help and exit + -p STRING, --prefix=STRING Use STRING as log file/profile prefix + -t NUMBER, --tail=NUMBER Only report the last NUMBER files + --title=TITLE Specify the output plot TITLE + """ + sys.stdout.write(self.outdent(help)) + sys.stdout.flush() + + def do_func(self, argv): + """ + """ + format = 'ascii' + function_name = '_main' + tail = None + + short_opts = '?C:f:hp:t:' + + long_opts = [ + 'chdir=', + 'file=', + 'fmt=', + 'format=', + 'func=', + 'function=', + 'help', + 'prefix=', + 'tail=', + 'title=', + ] + + opts, args = getopt.getopt(argv[1:], short_opts, long_opts) + + for o, a in opts: + if o in ('-C', '--chdir'): + self.chdir = a + elif o in ('-f', '--file'): + self.config_file = a + elif o in ('--fmt', '--format'): + format = a + elif o in ('--func', '--function'): + function_name = a + elif o in ('-?', '-h', '--help'): + self.do_help(['help', 'func']) + sys.exit(0) + elif o in ('--max',): + max_time = int(a) + elif o in ('-p', '--prefix'): + self.prefix = a + elif o in ('-t', '--tail'): + tail = int(a) + elif o in ('--title',): + self.title = a + + if self.config_file: + with open(self.config_file, 'r') as f: + config = f.read() + exec(config, self.__dict__) + + if self.chdir: + os.chdir(self.chdir) + + if not args: + + pattern = '%s*.prof' % self.prefix + args = self.args_to_files([pattern], tail) + + if not args: + if self.chdir: + directory = self.chdir + else: + directory = os.getcwd() + + sys.stderr.write('%s: func: No arguments specified.\n' % self.name) + sys.stderr.write('%s No %s*.prof files found in "%s".\n' % (self.name_spaces, self.prefix, directory)) + sys.stderr.write('%s Type "%s help func" for help.\n' % (self.name_spaces, self.name)) + sys.exit(1) + + else: + + args = self.args_to_files(args, tail) + + cwd_ = os.getcwd() + os.sep + + if format == 'ascii': + + for file in args: + try: + f, line, func, time = \ + self.get_function_profile(file, function_name) + except ValueError as e: + sys.stderr.write("%s: func: %s: %s\n" % + (self.name, file, e)) + else: + if f.startswith(cwd_): + f = f[len(cwd_):] + print("%.3f %s:%d(%s)" % (time, f, line, func)) + + elif format == 'gnuplot': + + results = self.collect_results(args, self.get_function_time, + function_name) + + self.gnuplot_results(results) + + else: + + sys.stderr.write('%s: func: Unknown format "%s".\n' % (self.name, format)) + sys.exit(1) + + # + + def help_mem(self): + help = """\ + Usage: scons-time mem [OPTIONS] FILE [...] + + -C DIR, --chdir=DIR Change to DIR before looking for files + -f FILE, --file=FILE Read configuration from specified FILE + --fmt=FORMAT, --format=FORMAT Print data in specified FORMAT + -h, --help Print this help and exit + -p STRING, --prefix=STRING Use STRING as log file/profile prefix + --stage=STAGE Plot memory at the specified stage: + pre-read, post-read, pre-build, + post-build (default: post-build) + -t NUMBER, --tail=NUMBER Only report the last NUMBER files + --title=TITLE Specify the output plot TITLE + """ + sys.stdout.write(self.outdent(help)) + sys.stdout.flush() + + def do_mem(self, argv): + + format = 'ascii' + logfile_path = lambda x: x + stage = self.default_stage + tail = None + + short_opts = '?C:f:hp:t:' + + long_opts = [ + 'chdir=', + 'file=', + 'fmt=', + 'format=', + 'help', + 'prefix=', + 'stage=', + 'tail=', + 'title=', + ] + + opts, args = getopt.getopt(argv[1:], short_opts, long_opts) + + for o, a in opts: + if o in ('-C', '--chdir'): + self.chdir = a + elif o in ('-f', '--file'): + self.config_file = a + elif o in ('--fmt', '--format'): + format = a + elif o in ('-?', '-h', '--help'): + self.do_help(['help', 'mem']) + sys.exit(0) + elif o in ('-p', '--prefix'): + self.prefix = a + elif o in ('--stage',): + if a not in self.stages: + sys.stderr.write('%s: mem: Unrecognized stage "%s".\n' % (self.name, a)) + sys.exit(1) + stage = a + elif o in ('-t', '--tail'): + tail = int(a) + elif o in ('--title',): + self.title = a + + if self.config_file: + with open(self.config_file, 'r') as f: + config = f.read() + HACK_for_exec(config, self.__dict__) + + if self.chdir: + os.chdir(self.chdir) + logfile_path = lambda x: os.path.join(self.chdir, x) + + if not args: + + pattern = '%s*.log' % self.prefix + args = self.args_to_files([pattern], tail) + + if not args: + if self.chdir: + directory = self.chdir + else: + directory = os.getcwd() + + sys.stderr.write('%s: mem: No arguments specified.\n' % self.name) + sys.stderr.write('%s No %s*.log files found in "%s".\n' % (self.name_spaces, self.prefix, directory)) + sys.stderr.write('%s Type "%s help mem" for help.\n' % (self.name_spaces, self.name)) + sys.exit(1) + + else: + + args = self.args_to_files(args, tail) + + cwd_ = os.getcwd() + os.sep + + if format == 'ascii': + + self.ascii_table(args, tuple(self.stages), self.get_memory, logfile_path) + + elif format == 'gnuplot': + + results = self.collect_results(args, self.get_memory, + self.stage_strings[stage]) + + self.gnuplot_results(results) + + else: + + sys.stderr.write('%s: mem: Unknown format "%s".\n' % (self.name, format)) + sys.exit(1) + + return 0 + + # + + def help_obj(self): + help = """\ + Usage: scons-time obj [OPTIONS] OBJECT FILE [...] + + -C DIR, --chdir=DIR Change to DIR before looking for files + -f FILE, --file=FILE Read configuration from specified FILE + --fmt=FORMAT, --format=FORMAT Print data in specified FORMAT + -h, --help Print this help and exit + -p STRING, --prefix=STRING Use STRING as log file/profile prefix + --stage=STAGE Plot memory at the specified stage: + pre-read, post-read, pre-build, + post-build (default: post-build) + -t NUMBER, --tail=NUMBER Only report the last NUMBER files + --title=TITLE Specify the output plot TITLE + """ + sys.stdout.write(self.outdent(help)) + sys.stdout.flush() + + def do_obj(self, argv): + + format = 'ascii' + logfile_path = lambda x: x + stage = self.default_stage + tail = None + + short_opts = '?C:f:hp:t:' + + long_opts = [ + 'chdir=', + 'file=', + 'fmt=', + 'format=', + 'help', + 'prefix=', + 'stage=', + 'tail=', + 'title=', + ] + + opts, args = getopt.getopt(argv[1:], short_opts, long_opts) + + for o, a in opts: + if o in ('-C', '--chdir'): + self.chdir = a + elif o in ('-f', '--file'): + self.config_file = a + elif o in ('--fmt', '--format'): + format = a + elif o in ('-?', '-h', '--help'): + self.do_help(['help', 'obj']) + sys.exit(0) + elif o in ('-p', '--prefix'): + self.prefix = a + elif o in ('--stage',): + if a not in self.stages: + sys.stderr.write('%s: obj: Unrecognized stage "%s".\n' % (self.name, a)) + sys.stderr.write('%s Type "%s help obj" for help.\n' % (self.name_spaces, self.name)) + sys.exit(1) + stage = a + elif o in ('-t', '--tail'): + tail = int(a) + elif o in ('--title',): + self.title = a + + if not args: + sys.stderr.write('%s: obj: Must specify an object name.\n' % self.name) + sys.stderr.write('%s Type "%s help obj" for help.\n' % (self.name_spaces, self.name)) + sys.exit(1) + + object_name = args.pop(0) + + if self.config_file: + with open(self.config_file, 'r') as f: + config = f.read() + HACK_for_exec(config, self.__dict__) + + if self.chdir: + os.chdir(self.chdir) + logfile_path = lambda x: os.path.join(self.chdir, x) + + if not args: + + pattern = '%s*.log' % self.prefix + args = self.args_to_files([pattern], tail) + + if not args: + if self.chdir: + directory = self.chdir + else: + directory = os.getcwd() + + sys.stderr.write('%s: obj: No arguments specified.\n' % self.name) + sys.stderr.write('%s No %s*.log files found in "%s".\n' % (self.name_spaces, self.prefix, directory)) + sys.stderr.write('%s Type "%s help obj" for help.\n' % (self.name_spaces, self.name)) + sys.exit(1) + + else: + + args = self.args_to_files(args, tail) + + cwd_ = os.getcwd() + os.sep + + if format == 'ascii': + + self.ascii_table(args, tuple(self.stages), self.get_object_counts, logfile_path, object_name) + + elif format == 'gnuplot': + + stage_index = 0 + for s in self.stages: + if stage == s: + break + stage_index = stage_index + 1 + + results = self.collect_results(args, self.get_object_counts, + object_name, stage_index) + + self.gnuplot_results(results) + + else: + + sys.stderr.write('%s: obj: Unknown format "%s".\n' % (self.name, format)) + sys.exit(1) + + return 0 + + # + + def help_run(self): + help = """\ + Usage: scons-time run [OPTIONS] [FILE ...] + + --chdir=DIR Name of unpacked directory for chdir + -f FILE, --file=FILE Read configuration from specified FILE + -h, --help Print this help and exit + -n, --no-exec No execute, just print command lines + --number=NUMBER Put output in files for run NUMBER + --outdir=OUTDIR Put output files in OUTDIR + -p STRING, --prefix=STRING Use STRING as log file/profile prefix + --python=PYTHON Time using the specified PYTHON + -q, --quiet Don't print command lines + --scons=SCONS Time using the specified SCONS + --svn=URL, --subversion=URL Use SCons from Subversion URL + -v, --verbose Display output of commands + """ + sys.stdout.write(self.outdent(help)) + sys.stdout.flush() + + def do_run(self, argv): + """ + """ + run_number_list = [None] + + short_opts = '?f:hnp:qs:v' + + long_opts = [ + 'file=', + 'help', + 'no-exec', + 'number=', + 'outdir=', + 'prefix=', + 'python=', + 'quiet', + 'scons=', + 'svn=', + 'subdir=', + 'subversion=', + 'verbose', + ] + + opts, args = getopt.getopt(argv[1:], short_opts, long_opts) + + for o, a in opts: + if o in ('-f', '--file'): + self.config_file = a + elif o in ('-?', '-h', '--help'): + self.do_help(['help', 'run']) + sys.exit(0) + elif o in ('-n', '--no-exec'): + self.execute = self._do_not_execute + elif o in ('--number',): + run_number_list = self.split_run_numbers(a) + elif o in ('--outdir',): + self.outdir = a + elif o in ('-p', '--prefix'): + self.prefix = a + elif o in ('--python',): + self.python = a + elif o in ('-q', '--quiet'): + self.display = self._do_not_display + elif o in ('-s', '--subdir'): + self.subdir = a + elif o in ('--scons',): + self.scons = a + elif o in ('--svn', '--subversion'): + self.subversion_url = a + elif o in ('-v', '--verbose'): + self.redirect = tee_to_file + self.verbose = True + self.svn_co_flag = '' + + if not args and not self.config_file: + sys.stderr.write('%s: run: No arguments or -f config file specified.\n' % self.name) + sys.stderr.write('%s Type "%s help run" for help.\n' % (self.name_spaces, self.name)) + sys.exit(1) + + if self.config_file: + with open(self.config_file, 'r') as f: + config = f.read() + exec(config, self.__dict__) + + if args: + self.archive_list = args + + archive_file_name = os.path.split(self.archive_list[0])[1] + + if not self.subdir: + self.subdir = self.archive_splitext(archive_file_name)[0] + + if not self.prefix: + self.prefix = self.archive_splitext(archive_file_name)[0] + + prepare = None + if self.subversion_url: + prepare = self.prep_subversion_run + + for run_number in run_number_list: + self.individual_run(run_number, self.archive_list, prepare) + + def split_run_numbers(self, s): + result = [] + for n in s.split(','): + try: + x, y = n.split('-') + except ValueError: + result.append(int(n)) + else: + result.extend(list(range(int(x), int(y)+1))) + return result + + def scons_path(self, dir): + return os.path.join(dir,'scripts', 'scons.py') + + def scons_lib_dir_path(self, dir): + return os.path.join(dir, 'src', 'engine') + + def prep_subversion_run(self, commands, removals): + self.svn_tmpdir = tempfile.mkdtemp(prefix=self.name + '-svn-') + removals.append((shutil.rmtree, 'rm -rf %%s', self.svn_tmpdir)) + + self.scons = self.scons_path(self.svn_tmpdir) + self.scons_lib_dir = self.scons_lib_dir_path(self.svn_tmpdir) + + commands.extend([ + '%(svn)s co %(svn_co_flag)s -r %(run_number)s %(subversion_url)s %(svn_tmpdir)s', + ]) + + def individual_run(self, run_number, archive_list, prepare=None): + """ + Performs an individual run of the default SCons invocations. + """ + + commands = [] + removals = [] + + if prepare: + prepare(commands, removals) + + save_scons = self.scons + save_scons_wrapper = self.scons_wrapper + save_scons_lib_dir = self.scons_lib_dir + + if self.outdir is None: + self.outdir = self.orig_cwd + elif not os.path.isabs(self.outdir): + self.outdir = os.path.join(self.orig_cwd, self.outdir) + + if self.scons is None: + self.scons = self.scons_path(self.orig_cwd) + + if self.scons_lib_dir is None: + self.scons_lib_dir = self.scons_lib_dir_path(self.orig_cwd) + + if self.scons_wrapper is None: + self.scons_wrapper = self.scons + + if not run_number: + run_number = self.find_next_run_number(self.outdir, self.prefix) + + self.run_number = str(run_number) + + self.prefix_run = self.prefix + '-%03d' % run_number + + if self.targets0 is None: + self.targets0 = self.startup_targets + if self.targets1 is None: + self.targets1 = self.targets + if self.targets2 is None: + self.targets2 = self.targets + + self.tmpdir = tempfile.mkdtemp(prefix=self.name + '-') + + commands.extend([ + (os.chdir, 'cd %%s', self.tmpdir), + ]) + + for archive in archive_list: + if not os.path.isabs(archive): + archive = os.path.join(self.orig_cwd, archive) + if os.path.isdir(archive): + dest = os.path.split(archive)[1] + commands.append((shutil.copytree, 'cp -r %%s %%s', archive, dest)) + else: + suffix = self.archive_splitext(archive)[1] + unpack_command = self.unpack_map.get(suffix) + if not unpack_command: + dest = os.path.split(archive)[1] + commands.append((shutil.copyfile, 'cp %%s %%s', archive, dest)) + else: + commands.append(unpack_command + (archive,)) + + commands.extend([ + (os.chdir, 'cd %%s', self.subdir), + ]) + + commands.extend(self.initial_commands) + + commands.extend([ + (lambda: read_tree('.'), + 'find * -type f | xargs cat > /dev/null'), + + (self.set_env, 'export %%s=%%s', + 'SCONS_LIB_DIR', self.scons_lib_dir), + + '%(python)s %(scons_wrapper)s --version', + ]) + + index = 0 + for run_command in self.run_commands: + setattr(self, 'prof%d' % index, self.profile_name(index)) + c = ( + self.log_execute, + self.log_display, + run_command, + self.logfile_name(index), + ) + commands.append(c) + index = index + 1 + + commands.extend([ + (os.chdir, 'cd %%s', self.orig_cwd), + ]) + + if not os.environ.get('PRESERVE'): + commands.extend(removals) + commands.append((shutil.rmtree, 'rm -rf %%s', self.tmpdir)) + + self.run_command_list(commands, self.__dict__) + + self.scons = save_scons + self.scons_lib_dir = save_scons_lib_dir + self.scons_wrapper = save_scons_wrapper + + # + + def help_time(self): + help = """\ + Usage: scons-time time [OPTIONS] FILE [...] + + -C DIR, --chdir=DIR Change to DIR before looking for files + -f FILE, --file=FILE Read configuration from specified FILE + --fmt=FORMAT, --format=FORMAT Print data in specified FORMAT + -h, --help Print this help and exit + -p STRING, --prefix=STRING Use STRING as log file/profile prefix + -t NUMBER, --tail=NUMBER Only report the last NUMBER files + --which=TIMER Plot timings for TIMER: total, + SConscripts, SCons, commands. + """ + sys.stdout.write(self.outdent(help)) + sys.stdout.flush() + + def do_time(self, argv): + + format = 'ascii' + logfile_path = lambda x: x + tail = None + which = 'total' + + short_opts = '?C:f:hp:t:' + + long_opts = [ + 'chdir=', + 'file=', + 'fmt=', + 'format=', + 'help', + 'prefix=', + 'tail=', + 'title=', + 'which=', + ] + + opts, args = getopt.getopt(argv[1:], short_opts, long_opts) + + for o, a in opts: + if o in ('-C', '--chdir'): + self.chdir = a + elif o in ('-f', '--file'): + self.config_file = a + elif o in ('--fmt', '--format'): + format = a + elif o in ('-?', '-h', '--help'): + self.do_help(['help', 'time']) + sys.exit(0) + elif o in ('-p', '--prefix'): + self.prefix = a + elif o in ('-t', '--tail'): + tail = int(a) + elif o in ('--title',): + self.title = a + elif o in ('--which',): + if a not in list(self.time_strings.keys()): + sys.stderr.write('%s: time: Unrecognized timer "%s".\n' % (self.name, a)) + sys.stderr.write('%s Type "%s help time" for help.\n' % (self.name_spaces, self.name)) + sys.exit(1) + which = a + + if self.config_file: + with open(self.config_file, 'r') as f: + config = f.read() + HACK_for_exec(config, self.__dict__) + + if self.chdir: + os.chdir(self.chdir) + logfile_path = lambda x: os.path.join(self.chdir, x) + + if not args: + + pattern = '%s*.log' % self.prefix + args = self.args_to_files([pattern], tail) + + if not args: + if self.chdir: + directory = self.chdir + else: + directory = os.getcwd() + + sys.stderr.write('%s: time: No arguments specified.\n' % self.name) + sys.stderr.write('%s No %s*.log files found in "%s".\n' % (self.name_spaces, self.prefix, directory)) + sys.stderr.write('%s Type "%s help time" for help.\n' % (self.name_spaces, self.name)) + sys.exit(1) + + else: + + args = self.args_to_files(args, tail) + + cwd_ = os.getcwd() + os.sep + + if format == 'ascii': + + columns = ("Total", "SConscripts", "SCons", "commands") + self.ascii_table(args, columns, self.get_debug_times, logfile_path) + + elif format == 'gnuplot': + + results = self.collect_results(args, self.get_debug_times, + self.time_strings[which]) + + self.gnuplot_results(results, fmt='%s %.6f') + + else: + + sys.stderr.write('%s: time: Unknown format "%s".\n' % (self.name, format)) + sys.exit(1) + +if __name__ == '__main__': + opts, args = getopt.getopt(sys.argv[1:], 'h?V', ['help', 'version']) + + ST = SConsTimer() + + for o, a in opts: + if o in ('-?', '-h', '--help'): + ST.do_help(['help']) + sys.exit(0) + elif o in ('-V', '--version'): + sys.stdout.write('scons-time version\n') + sys.exit(0) + + if not args: + sys.stderr.write('Type "%s help" for usage.\n' % ST.name) + sys.exit(1) + + ST.execute_subcommand(args) + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/scons-time/run/archive/dir.py b/test/scons-time/run/archive/dir.py index 590d568..6b6d992 100644 --- a/test/scons-time/run/archive/dir.py +++ b/test/scons-time/run/archive/dir.py @@ -32,6 +32,7 @@ directory tree. import TestSCons_time test = TestSCons_time.TestSCons_time() +test.verbose_set(1) test.write_fake_scons_py() diff --git a/test/scons-time/run/config/targets.py b/test/scons-time/run/config/targets.py index 482a44d..1712b32 100644 --- a/test/scons-time/run/config/targets.py +++ b/test/scons-time/run/config/targets.py @@ -45,7 +45,7 @@ targets = 'target1 target2' test.run(arguments = 'run -f config foo.tar.gz') -scons_py = re.escape(test.workpath('src', 'script', 'scons.py')) +scons_py = re.escape(test.workpath('scripts', 'scons.py')) src_engine = re.escape(test.workpath('src', 'engine')) prof1 = re.escape(test.workpath('foo-000-1.prof')) diff --git a/test/scons-time/run/option/quiet.py b/test/scons-time/run/option/quiet.py index 2910e8e..0f6c22b 100644 --- a/test/scons-time/run/option/quiet.py +++ b/test/scons-time/run/option/quiet.py @@ -37,7 +37,7 @@ python = TestSCons_time.python test = TestSCons_time.TestSCons_time(match = TestSCons_time.match_re, diff = TestSCons_time.diff_re) -scons_py = re.escape(test.workpath('src', 'script', 'scons.py')) +scons_py = re.escape(test.workpath('scripts', 'scons.py')) src_engine = re.escape(test.workpath('src', 'engine')) tmp_scons_time = test.tempdir_re() diff --git a/test/scons-time/run/option/verbose.py b/test/scons-time/run/option/verbose.py index 6f8f72f..9c4e7e8 100644 --- a/test/scons-time/run/option/verbose.py +++ b/test/scons-time/run/option/verbose.py @@ -39,7 +39,7 @@ _python_ = re.escape('"' + sys.executable + '"') test = TestSCons_time.TestSCons_time(match = TestSCons_time.match_re, diff = TestSCons_time.diff_re) -scons_py = re.escape(test.workpath('src', 'script', 'scons.py')) +scons_py = re.escape(test.workpath('scripts', 'scons.py')) src_engine = re.escape(test.workpath('src', 'engine')) tmp_scons_time = test.tempdir_re() diff --git a/test/scons-time/run/subversion.py b/test/scons-time/run/subversion.py deleted file mode 100644 index f895760..0000000 --- a/test/scons-time/run/subversion.py +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env python -# -# __COPYRIGHT__ -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - -""" -Verify ability to "check out" an SCons revision from a fake -Subversion utility. -""" - -import re -import tempfile - -import TestSCons_time - -test = TestSCons_time.TestSCons_time() - -test.write_sample_project('foo.tar') - -_python_ = TestSCons_time._python_ -my_svn_py = test.write_fake_svn_py('my_svn.py') - -test.write('config', """\ -svn = r'%(_python_)s %(my_svn_py)s' -""" % locals()) - -test.run(arguments = 'run -f config --svn http://xyzzy --number 617,716 foo.tar') - -test.must_exist('foo-617-0.log', - 'foo-617-0.prof', - 'foo-617-1.log', - 'foo-617-1.prof', - 'foo-617-2.log', - 'foo-617-2.prof') - -test.must_exist('foo-716-0.log', - 'foo-716-0.prof', - 'foo-716-1.log', - 'foo-716-1.prof', - 'foo-716-2.log', - 'foo-716-2.prof') - -expect = [ - test.tempdir_re('src', 'script', 'scons.py'), - 'SCONS_LIB_DIR = %s' % test.tempdir_re('src', 'engine'), -] - -content = test.read(test.workpath('foo-617-2.log'), mode='r') - -def re_find(content, line): - return re.search(line, content) -test.must_contain_all_lines(content, expect, 'foo-617-2.log', re_find) - -test.pass_test() - -# Local Variables: -# tab-width:4 -# indent-tabs-mode:nil -# End: -# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/testing/framework/TestSCons_time.py b/testing/framework/TestSCons_time.py index 6f46e26..c6373eb 100644 --- a/testing/framework/TestSCons_time.py +++ b/testing/framework/TestSCons_time.py @@ -60,7 +60,7 @@ import os import sys dir = sys.argv[-1] -script_dir = dir + '/src/script' +script_dir = dir + '/scripts' os.makedirs(script_dir) with open(script_dir + '/scons.py', 'w') as f: f.write(r'''%s''') @@ -73,7 +73,7 @@ import os import sys dir = sys.argv[-1] -script_dir = dir + '/src/script' +script_dir = dir + '/scripts' os.makedirs(script_dir) with open(script_dir + '/scons.py', 'w') as f: f.write(r'''%s''') @@ -230,8 +230,8 @@ class TestSCons_time(TestCommon): return x def write_fake_scons_py(self): - self.subdir('src', ['src', 'script']) - self.write('src/script/scons.py', scons_py) + self.subdir('scripts') + self.write('scripts/scons.py', scons_py) def write_fake_svn_py(self, name): name = self.workpath(name) -- cgit v0.12 From 5e8c9536ec9f76901fcc02cc716fd9ccf2d9aaa5 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 21 Mar 2020 19:50:04 -0700 Subject: Fix sconsign tests and script --- src/engine/SCons/Utilities/sconsign.py | 49 +++------------------------------- test/sconsign/script/SConsignFile.py | 1 - 2 files changed, 4 insertions(+), 46 deletions(-) diff --git a/src/engine/SCons/Utilities/sconsign.py b/src/engine/SCons/Utilities/sconsign.py index 819b9f7..8b16b18 100644 --- a/src/engine/SCons/Utilities/sconsign.py +++ b/src/engine/SCons/Utilities/sconsign.py @@ -43,50 +43,6 @@ from dbm import whichdb import time import pickle -# python compatibility check -if sys.version_info < (3, 5, 0): - msg = "scons: *** SCons version %s does not run under Python version %s.\n\ -Python >= 3.5 is required.\n" - sys.stderr.write(msg % (__version__, sys.version.split()[0])) - sys.exit(1) - -# Strip the script directory from sys.path so on case-insensitive -# (WIN32) systems Python doesn't think that the "scons" script is the -# "SCons" package. -script_dir = os.path.dirname(os.path.realpath(__file__)) -script_path = os.path.realpath(os.path.dirname(__file__)) -if script_path in sys.path: - sys.path.remove(script_path) - -libs = [] - -if "SCONS_LIB_DIR" in os.environ: - libs.append(os.environ["SCONS_LIB_DIR"]) - -# running from source takes 2nd priority (since 2.3.2), following SCONS_LIB_DIR -source_path = os.path.join(script_path, os.pardir, 'src', 'engine') -if os.path.isdir(source_path): - libs.append(source_path) - -# add local-install locations -local_version = 'scons-local-' + __version__ -local = 'scons-local' -if script_dir: - local_version = os.path.join(script_dir, local_version) - local = os.path.join(script_dir, local) -if os.path.isdir(local_version): - libs.append(os.path.abspath(local_version)) -if os.path.isdir(local): - libs.append(os.path.abspath(local)) - -scons_version = 'scons-%s' % __version__ - -sys.path = libs + sys.path - -############################################################################## -# END STANDARD SCons SCRIPT HEADER -############################################################################## - import SCons.compat import SCons.SConsign @@ -429,6 +385,10 @@ def Do_SConsignDir(name): ############################################################################## def main(): global Do_Call + global nodeinfo_string + global args + global Verbose + global Readable helpstr = """\ Usage: sconsign [OPTIONS] [FILE ...] @@ -536,7 +496,6 @@ def main(): print("NOTE: there were %d warnings, please check output" % Warns) - if __name__ == "__main__": main() sys.exit(0) diff --git a/test/sconsign/script/SConsignFile.py b/test/sconsign/script/SConsignFile.py index 8055b68..cd82fac 100644 --- a/test/sconsign/script/SConsignFile.py +++ b/test/sconsign/script/SConsignFile.py @@ -36,7 +36,6 @@ import TestSConsign _python_ = TestSCons._python_ test = TestSConsign.TestSConsign(match = TestSConsign.match_re) - test.subdir('sub1', 'sub2') fake_cc_py = test.workpath('fake_cc.py') -- cgit v0.12 From d3acf86b170be29de6b0328b063f5451e1f7e917 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 21 Mar 2020 20:05:46 -0700 Subject: Fix tests. IMPLICIT_COMMAND_DEPENDENCIES=all needs to be quoted --- test/Repository/Program.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Repository/Program.py b/test/Repository/Program.py index 1af3578..3946d95 100644 --- a/test/Repository/Program.py +++ b/test/Repository/Program.py @@ -32,7 +32,7 @@ if sys.platform == 'win32': else: _exe = '' -for implicit_deps in ['0', '1', '"all"']: +for implicit_deps in ['0', '1', '2', '\"all\"']: # First, test a single repository. test = TestSCons.TestSCons() test.subdir('repository', 'work1') -- cgit v0.12 From e9fa3ea7c3e57feded1d200b88650629cccae885 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 21 Mar 2020 20:02:33 -0700 Subject: Fix test --- test/packaging/rpm/explicit-target.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/packaging/rpm/explicit-target.py b/test/packaging/rpm/explicit-target.py index fba0237..0cf486e 100644 --- a/test/packaging/rpm/explicit-target.py +++ b/test/packaging/rpm/explicit-target.py @@ -77,7 +77,7 @@ env.Package( NAME = 'foo', expect = """ scons: *** Setting target is not supported for rpm. -""" + test.python_file_line(test.workpath('SConstruct'), 23) +""" + test.python_file_line(test.workpath('SConstruct'), 12) test.run(arguments='', status=2, stderr=expect) -- cgit v0.12 From 269341a2cc10c1a789d21232ce94ad2b1fceab25 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sun, 22 Mar 2020 12:44:08 -0700 Subject: drop runntest.py's run from packages. define SCONS_TOOLS_DIR and use that for scons-time tests --- bin/scons-time.py | 1480 +++++++++++++++++++++++++++++++++++ runtest.py | 116 +-- scripts/scons-time.py | 1480 ----------------------------------- testing/framework/TestSCons_time.py | 4 +- 4 files changed, 1519 insertions(+), 1561 deletions(-) create mode 100644 bin/scons-time.py delete mode 100644 scripts/scons-time.py diff --git a/bin/scons-time.py b/bin/scons-time.py new file mode 100644 index 0000000..6494349 --- /dev/null +++ b/bin/scons-time.py @@ -0,0 +1,1480 @@ +#!/usr/bin/env python +# +# scons-time - run SCons timings and collect statistics +# +# A script for running a configuration through SCons with a standard +# set of invocations to collect timing and memory statistics and to +# capture the results in a consistent set of output files for display +# and analysis. +# + +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +import getopt +import glob +import os +import re +import shutil +import sys +import tempfile +import time +import subprocess + +def HACK_for_exec(cmd, *args): + """ + For some reason, Python won't allow an exec() within a function + that also declares an internal function (including lambda functions). + This function is a hack that calls exec() in a function with no + internal functions. + """ + if not args: exec(cmd) + elif len(args) == 1: exec(cmd, args[0]) + else: exec(cmd, args[0], args[1]) + +class Plotter(object): + def increment_size(self, largest): + """ + Return the size of each horizontal increment line for a specified + maximum value. This returns a value that will provide somewhere + between 5 and 9 horizontal lines on the graph, on some set of + boundaries that are multiples of 10/100/1000/etc. + """ + i = largest // 5 + if not i: + return largest + multiplier = 1 + while i >= 10: + i = i // 10 + multiplier = multiplier * 10 + return i * multiplier + + def max_graph_value(self, largest): + # Round up to next integer. + largest = int(largest) + 1 + increment = self.increment_size(largest) + return ((largest + increment - 1) // increment) * increment + +class Line(object): + def __init__(self, points, type, title, label, comment, fmt="%s %s"): + self.points = points + self.type = type + self.title = title + self.label = label + self.comment = comment + self.fmt = fmt + + def print_label(self, inx, x, y): + if self.label: + print('set label %s "%s" at %0.1f,%0.1f right' % (inx, self.label, x, y)) + + def plot_string(self): + if self.title: + title_string = 'title "%s"' % self.title + else: + title_string = 'notitle' + return "'-' %s with lines lt %s" % (title_string, self.type) + + def print_points(self, fmt=None): + if fmt is None: + fmt = self.fmt + if self.comment: + print('# %s' % self.comment) + for x, y in self.points: + # If y is None, it usually represents some kind of break + # in the line's index number. We might want to represent + # this some way rather than just drawing the line straight + # between the two points on either side. + if y is not None: + print(fmt % (x, y)) + print('e') + + def get_x_values(self): + return [ p[0] for p in self.points ] + + def get_y_values(self): + return [ p[1] for p in self.points ] + +class Gnuplotter(Plotter): + + def __init__(self, title, key_location): + self.lines = [] + self.title = title + self.key_location = key_location + + def line(self, points, type, title=None, label=None, comment=None, fmt='%s %s'): + if points: + line = Line(points, type, title, label, comment, fmt) + self.lines.append(line) + + def plot_string(self, line): + return line.plot_string() + + def vertical_bar(self, x, type, label, comment): + if self.get_min_x() <= x <= self.get_max_x(): + points = [(x, 0), (x, self.max_graph_value(self.get_max_y()))] + self.line(points, type, label, comment) + + def get_all_x_values(self): + result = [] + for line in self.lines: + result.extend(line.get_x_values()) + return [r for r in result if r is not None] + + def get_all_y_values(self): + result = [] + for line in self.lines: + result.extend(line.get_y_values()) + return [r for r in result if r is not None] + + def get_min_x(self): + try: + return self.min_x + except AttributeError: + try: + self.min_x = min(self.get_all_x_values()) + except ValueError: + self.min_x = 0 + return self.min_x + + def get_max_x(self): + try: + return self.max_x + except AttributeError: + try: + self.max_x = max(self.get_all_x_values()) + except ValueError: + self.max_x = 0 + return self.max_x + + def get_min_y(self): + try: + return self.min_y + except AttributeError: + try: + self.min_y = min(self.get_all_y_values()) + except ValueError: + self.min_y = 0 + return self.min_y + + def get_max_y(self): + try: + return self.max_y + except AttributeError: + try: + self.max_y = max(self.get_all_y_values()) + except ValueError: + self.max_y = 0 + return self.max_y + + def draw(self): + + if not self.lines: + return + + if self.title: + print('set title "%s"' % self.title) + print('set key %s' % self.key_location) + + min_y = self.get_min_y() + max_y = self.max_graph_value(self.get_max_y()) + incr = (max_y - min_y) / 10.0 + start = min_y + (max_y / 2.0) + (2.0 * incr) + position = [ start - (i * incr) for i in range(5) ] + + inx = 1 + for line in self.lines: + line.print_label(inx, line.points[0][0]-1, + position[(inx-1) % len(position)]) + inx += 1 + + plot_strings = [ self.plot_string(l) for l in self.lines ] + print('plot ' + ', \\\n '.join(plot_strings)) + + for line in self.lines: + line.print_points() + + + +def untar(fname): + import tarfile + tar = tarfile.open(name=fname, mode='r') + for tarinfo in tar: + tar.extract(tarinfo) + tar.close() + +def unzip(fname): + import zipfile + zf = zipfile.ZipFile(fname, 'r') + for name in zf.namelist(): + dir = os.path.dirname(name) + try: + os.makedirs(dir) + except: + pass + with open(name, 'wb') as f: + f.write(zf.read(name)) + +def read_tree(dir): + for dirpath, dirnames, filenames in os.walk(dir): + for fn in filenames: + fn = os.path.join(dirpath, fn) + if os.path.isfile(fn): + with open(fn, 'rb') as f: + f.read() + +def redirect_to_file(command, log): + return '%s > %s 2>&1' % (command, log) + +def tee_to_file(command, log): + return '%s 2>&1 | tee %s' % (command, log) + + + +class SConsTimer(object): + """ + Usage: scons-time SUBCOMMAND [ARGUMENTS] + Type "scons-time help SUBCOMMAND" for help on a specific subcommand. + + Available subcommands: + func Extract test-run data for a function + help Provides help + mem Extract --debug=memory data from test runs + obj Extract --debug=count data from test runs + time Extract --debug=time data from test runs + run Runs a test configuration + """ + + name = 'scons-time' + name_spaces = ' '*len(name) + + def makedict(**kw): + return kw + + default_settings = makedict( + chdir = None, + config_file = None, + initial_commands = [], + key_location = 'bottom left', + orig_cwd = os.getcwd(), + outdir = None, + prefix = '', + python = '"%s"' % sys.executable, + redirect = redirect_to_file, + scons = None, + scons_flags = '--debug=count --debug=memory --debug=time --debug=memoizer', + scons_lib_dir = None, + scons_wrapper = None, + startup_targets = '--help', + subdir = None, + subversion_url = None, + svn = 'svn', + svn_co_flag = '-q', + tar = 'tar', + targets = '', + targets0 = None, + targets1 = None, + targets2 = None, + title = None, + unzip = 'unzip', + verbose = False, + vertical_bars = [], + + unpack_map = { + '.tar.gz' : (untar, '%(tar)s xzf %%s'), + '.tgz' : (untar, '%(tar)s xzf %%s'), + '.tar' : (untar, '%(tar)s xf %%s'), + '.zip' : (unzip, '%(unzip)s %%s'), + }, + ) + + run_titles = [ + 'Startup', + 'Full build', + 'Up-to-date build', + ] + + run_commands = [ + '%(python)s %(scons_wrapper)s %(scons_flags)s --profile=%(prof0)s %(targets0)s', + '%(python)s %(scons_wrapper)s %(scons_flags)s --profile=%(prof1)s %(targets1)s', + '%(python)s %(scons_wrapper)s %(scons_flags)s --profile=%(prof2)s %(targets2)s', + ] + + stages = [ + 'pre-read', + 'post-read', + 'pre-build', + 'post-build', + ] + + stage_strings = { + 'pre-read' : 'Memory before reading SConscript files:', + 'post-read' : 'Memory after reading SConscript files:', + 'pre-build' : 'Memory before building targets:', + 'post-build' : 'Memory after building targets:', + } + + memory_string_all = 'Memory ' + + default_stage = stages[-1] + + time_strings = { + 'total' : 'Total build time', + 'SConscripts' : 'Total SConscript file execution time', + 'SCons' : 'Total SCons execution time', + 'commands' : 'Total command execution time', + } + + time_string_all = 'Total .* time' + + # + + def __init__(self): + self.__dict__.update(self.default_settings) + + # Functions for displaying and executing commands. + + def subst(self, x, dictionary): + try: + return x % dictionary + except TypeError: + # x isn't a string (it's probably a Python function), + # so just return it. + return x + + def subst_variables(self, command, dictionary): + """ + Substitutes (via the format operator) the values in the specified + dictionary into the specified command. + + The command can be an (action, string) tuple. In all cases, we + perform substitution on strings and don't worry if something isn't + a string. (It's probably a Python function to be executed.) + """ + try: + command + '' + except TypeError: + action = command[0] + string = command[1] + args = command[2:] + else: + action = command + string = action + args = (()) + action = self.subst(action, dictionary) + string = self.subst(string, dictionary) + return (action, string, args) + + def _do_not_display(self, msg, *args): + pass + + def display(self, msg, *args): + """ + Displays the specified message. + + Each message is prepended with a standard prefix of our name + plus the time. + """ + if callable(msg): + msg = msg(*args) + else: + msg = msg % args + if msg is None: + return + fmt = '%s[%s]: %s\n' + sys.stdout.write(fmt % (self.name, time.strftime('%H:%M:%S'), msg)) + + def _do_not_execute(self, action, *args): + pass + + def execute(self, action, *args): + """ + Executes the specified action. + + The action is called if it's a callable Python function, and + otherwise passed to os.system(). + """ + if callable(action): + action(*args) + else: + os.system(action % args) + + def run_command_list(self, commands, dict): + """ + Executes a list of commands, substituting values from the + specified dictionary. + """ + commands = [ self.subst_variables(c, dict) for c in commands ] + for action, string, args in commands: + self.display(string, *args) + sys.stdout.flush() + status = self.execute(action, *args) + if status: + sys.exit(status) + + def log_display(self, command, log): + command = self.subst(command, self.__dict__) + if log: + command = self.redirect(command, log) + return command + + def log_execute(self, command, log): + command = self.subst(command, self.__dict__) + p = os.popen(command) + output = p.read() + p.close() + #TODO: convert to subrocess, os.popen is obsolete. This didn't work: + #process = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True) + #output = process.stdout.read() + #process.stdout.close() + #process.wait() + if self.verbose: + sys.stdout.write(output) + # TODO: Figure out + # Not sure we need to write binary here + with open(log, 'w') as f: + f.write(str(output)) + + def archive_splitext(self, path): + """ + Splits an archive name into a filename base and extension. + + This is like os.path.splitext() (which it calls) except that it + also looks for '.tar.gz' and treats it as an atomic extensions. + """ + if path.endswith('.tar.gz'): + return path[:-7], path[-7:] + else: + return os.path.splitext(path) + + def args_to_files(self, args, tail=None): + """ + Takes a list of arguments, expands any glob patterns, and + returns the last "tail" files from the list. + """ + files = [] + for a in args: + files.extend(sorted(glob.glob(a))) + + if tail: + files = files[-tail:] + + return files + + def ascii_table(self, files, columns, + line_function, file_function=lambda x: x, + *args, **kw): + + header_fmt = ' '.join(['%12s'] * len(columns)) + line_fmt = header_fmt + ' %s' + + print(header_fmt % columns) + + for file in files: + t = line_function(file, *args, **kw) + if t is None: + t = [] + diff = len(columns) - len(t) + if diff > 0: + t += [''] * diff + t.append(file_function(file)) + print(line_fmt % tuple(t)) + + def collect_results(self, files, function, *args, **kw): + results = {} + + for file in files: + base = os.path.splitext(file)[0] + run, index = base.split('-')[-2:] + + run = int(run) + index = int(index) + + value = function(file, *args, **kw) + + try: + r = results[index] + except KeyError: + r = [] + results[index] = r + r.append((run, value)) + + return results + + def doc_to_help(self, obj): + """ + Translates an object's __doc__ string into help text. + + This strips a consistent number of spaces from each line in the + help text, essentially "outdenting" the text to the left-most + column. + """ + doc = obj.__doc__ + if doc is None: + return '' + return self.outdent(doc) + + def find_next_run_number(self, dir, prefix): + """ + Returns the next run number in a directory for the specified prefix. + + Examines the contents the specified directory for files with the + specified prefix, extracts the run numbers from each file name, + and returns the next run number after the largest it finds. + """ + x = re.compile(re.escape(prefix) + '-([0-9]+).*') + matches = [x.match(e) for e in os.listdir(dir)] + matches = [_f for _f in matches if _f] + if not matches: + return 0 + run_numbers = [int(m.group(1)) for m in matches] + return int(max(run_numbers)) + 1 + + def gnuplot_results(self, results, fmt='%s %.3f'): + """ + Prints out a set of results in Gnuplot format. + """ + gp = Gnuplotter(self.title, self.key_location) + + for i in sorted(results.keys()): + try: + t = self.run_titles[i] + except IndexError: + t = '??? %s ???' % i + results[i].sort() + gp.line(results[i], i+1, t, None, t, fmt=fmt) + + for bar_tuple in self.vertical_bars: + try: + x, type, label, comment = bar_tuple + except ValueError: + x, type, label = bar_tuple + comment = label + gp.vertical_bar(x, type, label, comment) + + gp.draw() + + def logfile_name(self, invocation): + """ + Returns the absolute path of a log file for the specificed + invocation number. + """ + name = self.prefix_run + '-%d.log' % invocation + return os.path.join(self.outdir, name) + + def outdent(self, s): + """ + Strip as many spaces from each line as are found at the beginning + of the first line in the list. + """ + lines = s.split('\n') + if lines[0] == '': + lines = lines[1:] + spaces = re.match(' *', lines[0]).group(0) + def strip_initial_spaces(l, s=spaces): + if l.startswith(spaces): + l = l[len(spaces):] + return l + return '\n'.join([ strip_initial_spaces(l) for l in lines ]) + '\n' + + def profile_name(self, invocation): + """ + Returns the absolute path of a profile file for the specified + invocation number. + """ + name = self.prefix_run + '-%d.prof' % invocation + return os.path.join(self.outdir, name) + + def set_env(self, key, value): + os.environ[key] = value + + # + + def get_debug_times(self, file, time_string=None): + """ + Fetch times from the --debug=time strings in the specified file. + """ + if time_string is None: + search_string = self.time_string_all + else: + search_string = time_string + with open(file) as f: + contents = f.read() + if not contents: + sys.stderr.write('file %s has no contents!\n' % repr(file)) + return None + result = re.findall(r'%s: ([\d.]*)' % search_string, contents)[-4:] + result = [ float(r) for r in result ] + if time_string is not None: + try: + result = result[0] + except IndexError: + sys.stderr.write('file %s has no results!\n' % repr(file)) + return None + return result + + def get_function_profile(self, file, function): + """ + Returns the file, line number, function name, and cumulative time. + """ + try: + import pstats + except ImportError as e: + sys.stderr.write('%s: func: %s\n' % (self.name, e)) + sys.stderr.write('%s This version of Python is missing the profiler.\n' % self.name_spaces) + sys.stderr.write('%s Cannot use the "func" subcommand.\n' % self.name_spaces) + sys.exit(1) + statistics = pstats.Stats(file).stats + matches = [ e for e in statistics.items() if e[0][2] == function ] + r = matches[0] + return r[0][0], r[0][1], r[0][2], r[1][3] + + def get_function_time(self, file, function): + """ + Returns just the cumulative time for the specified function. + """ + return self.get_function_profile(file, function)[3] + + def get_memory(self, file, memory_string=None): + """ + Returns a list of integers of the amount of memory used. The + default behavior is to return all the stages. + """ + if memory_string is None: + search_string = self.memory_string_all + else: + search_string = memory_string + with open(file) as f: + lines = f.readlines() + lines = [ l for l in lines if l.startswith(search_string) ][-4:] + result = [ int(l.split()[-1]) for l in lines[-4:] ] + if len(result) == 1: + result = result[0] + return result + + def get_object_counts(self, file, object_name, index=None): + """ + Returns the counts of the specified object_name. + """ + object_string = ' ' + object_name + '\n' + with open(file) as f: + lines = f.readlines() + line = [ l for l in lines if l.endswith(object_string) ][0] + result = [ int(field) for field in line.split()[:4] ] + if index is not None: + result = result[index] + return result + + + command_alias = {} + + def execute_subcommand(self, argv): + """ + Executes the do_*() function for the specified subcommand (argv[0]). + """ + if not argv: + return + cmdName = self.command_alias.get(argv[0], argv[0]) + try: + func = getattr(self, 'do_' + cmdName) + except AttributeError: + return self.default(argv) + try: + return func(argv) + except TypeError as e: + sys.stderr.write("%s %s: %s\n" % (self.name, cmdName, e)) + import traceback + traceback.print_exc(file=sys.stderr) + sys.stderr.write("Try '%s help %s'\n" % (self.name, cmdName)) + + def default(self, argv): + """ + The default behavior for an unknown subcommand. Prints an + error message and exits. + """ + sys.stderr.write('%s: Unknown subcommand "%s".\n' % (self.name, argv[0])) + sys.stderr.write('Type "%s help" for usage.\n' % self.name) + sys.exit(1) + + # + + def do_help(self, argv): + """ + """ + if argv[1:]: + for arg in argv[1:]: + try: + func = getattr(self, 'do_' + arg) + except AttributeError: + sys.stderr.write('%s: No help for "%s"\n' % (self.name, arg)) + else: + try: + help = getattr(self, 'help_' + arg) + except AttributeError: + sys.stdout.write(self.doc_to_help(func)) + sys.stdout.flush() + else: + help() + else: + doc = self.doc_to_help(self.__class__) + if doc: + sys.stdout.write(doc) + sys.stdout.flush() + return None + + # + + def help_func(self): + help = """\ + Usage: scons-time func [OPTIONS] FILE [...] + + -C DIR, --chdir=DIR Change to DIR before looking for files + -f FILE, --file=FILE Read configuration from specified FILE + --fmt=FORMAT, --format=FORMAT Print data in specified FORMAT + --func=NAME, --function=NAME Report time for function NAME + -h, --help Print this help and exit + -p STRING, --prefix=STRING Use STRING as log file/profile prefix + -t NUMBER, --tail=NUMBER Only report the last NUMBER files + --title=TITLE Specify the output plot TITLE + """ + sys.stdout.write(self.outdent(help)) + sys.stdout.flush() + + def do_func(self, argv): + """ + """ + format = 'ascii' + function_name = '_main' + tail = None + + short_opts = '?C:f:hp:t:' + + long_opts = [ + 'chdir=', + 'file=', + 'fmt=', + 'format=', + 'func=', + 'function=', + 'help', + 'prefix=', + 'tail=', + 'title=', + ] + + opts, args = getopt.getopt(argv[1:], short_opts, long_opts) + + for o, a in opts: + if o in ('-C', '--chdir'): + self.chdir = a + elif o in ('-f', '--file'): + self.config_file = a + elif o in ('--fmt', '--format'): + format = a + elif o in ('--func', '--function'): + function_name = a + elif o in ('-?', '-h', '--help'): + self.do_help(['help', 'func']) + sys.exit(0) + elif o in ('--max',): + max_time = int(a) + elif o in ('-p', '--prefix'): + self.prefix = a + elif o in ('-t', '--tail'): + tail = int(a) + elif o in ('--title',): + self.title = a + + if self.config_file: + with open(self.config_file, 'r') as f: + config = f.read() + exec(config, self.__dict__) + + if self.chdir: + os.chdir(self.chdir) + + if not args: + + pattern = '%s*.prof' % self.prefix + args = self.args_to_files([pattern], tail) + + if not args: + if self.chdir: + directory = self.chdir + else: + directory = os.getcwd() + + sys.stderr.write('%s: func: No arguments specified.\n' % self.name) + sys.stderr.write('%s No %s*.prof files found in "%s".\n' % (self.name_spaces, self.prefix, directory)) + sys.stderr.write('%s Type "%s help func" for help.\n' % (self.name_spaces, self.name)) + sys.exit(1) + + else: + + args = self.args_to_files(args, tail) + + cwd_ = os.getcwd() + os.sep + + if format == 'ascii': + + for file in args: + try: + f, line, func, time = \ + self.get_function_profile(file, function_name) + except ValueError as e: + sys.stderr.write("%s: func: %s: %s\n" % + (self.name, file, e)) + else: + if f.startswith(cwd_): + f = f[len(cwd_):] + print("%.3f %s:%d(%s)" % (time, f, line, func)) + + elif format == 'gnuplot': + + results = self.collect_results(args, self.get_function_time, + function_name) + + self.gnuplot_results(results) + + else: + + sys.stderr.write('%s: func: Unknown format "%s".\n' % (self.name, format)) + sys.exit(1) + + # + + def help_mem(self): + help = """\ + Usage: scons-time mem [OPTIONS] FILE [...] + + -C DIR, --chdir=DIR Change to DIR before looking for files + -f FILE, --file=FILE Read configuration from specified FILE + --fmt=FORMAT, --format=FORMAT Print data in specified FORMAT + -h, --help Print this help and exit + -p STRING, --prefix=STRING Use STRING as log file/profile prefix + --stage=STAGE Plot memory at the specified stage: + pre-read, post-read, pre-build, + post-build (default: post-build) + -t NUMBER, --tail=NUMBER Only report the last NUMBER files + --title=TITLE Specify the output plot TITLE + """ + sys.stdout.write(self.outdent(help)) + sys.stdout.flush() + + def do_mem(self, argv): + + format = 'ascii' + logfile_path = lambda x: x + stage = self.default_stage + tail = None + + short_opts = '?C:f:hp:t:' + + long_opts = [ + 'chdir=', + 'file=', + 'fmt=', + 'format=', + 'help', + 'prefix=', + 'stage=', + 'tail=', + 'title=', + ] + + opts, args = getopt.getopt(argv[1:], short_opts, long_opts) + + for o, a in opts: + if o in ('-C', '--chdir'): + self.chdir = a + elif o in ('-f', '--file'): + self.config_file = a + elif o in ('--fmt', '--format'): + format = a + elif o in ('-?', '-h', '--help'): + self.do_help(['help', 'mem']) + sys.exit(0) + elif o in ('-p', '--prefix'): + self.prefix = a + elif o in ('--stage',): + if a not in self.stages: + sys.stderr.write('%s: mem: Unrecognized stage "%s".\n' % (self.name, a)) + sys.exit(1) + stage = a + elif o in ('-t', '--tail'): + tail = int(a) + elif o in ('--title',): + self.title = a + + if self.config_file: + with open(self.config_file, 'r') as f: + config = f.read() + HACK_for_exec(config, self.__dict__) + + if self.chdir: + os.chdir(self.chdir) + logfile_path = lambda x: os.path.join(self.chdir, x) + + if not args: + + pattern = '%s*.log' % self.prefix + args = self.args_to_files([pattern], tail) + + if not args: + if self.chdir: + directory = self.chdir + else: + directory = os.getcwd() + + sys.stderr.write('%s: mem: No arguments specified.\n' % self.name) + sys.stderr.write('%s No %s*.log files found in "%s".\n' % (self.name_spaces, self.prefix, directory)) + sys.stderr.write('%s Type "%s help mem" for help.\n' % (self.name_spaces, self.name)) + sys.exit(1) + + else: + + args = self.args_to_files(args, tail) + + cwd_ = os.getcwd() + os.sep + + if format == 'ascii': + + self.ascii_table(args, tuple(self.stages), self.get_memory, logfile_path) + + elif format == 'gnuplot': + + results = self.collect_results(args, self.get_memory, + self.stage_strings[stage]) + + self.gnuplot_results(results) + + else: + + sys.stderr.write('%s: mem: Unknown format "%s".\n' % (self.name, format)) + sys.exit(1) + + return 0 + + # + + def help_obj(self): + help = """\ + Usage: scons-time obj [OPTIONS] OBJECT FILE [...] + + -C DIR, --chdir=DIR Change to DIR before looking for files + -f FILE, --file=FILE Read configuration from specified FILE + --fmt=FORMAT, --format=FORMAT Print data in specified FORMAT + -h, --help Print this help and exit + -p STRING, --prefix=STRING Use STRING as log file/profile prefix + --stage=STAGE Plot memory at the specified stage: + pre-read, post-read, pre-build, + post-build (default: post-build) + -t NUMBER, --tail=NUMBER Only report the last NUMBER files + --title=TITLE Specify the output plot TITLE + """ + sys.stdout.write(self.outdent(help)) + sys.stdout.flush() + + def do_obj(self, argv): + + format = 'ascii' + logfile_path = lambda x: x + stage = self.default_stage + tail = None + + short_opts = '?C:f:hp:t:' + + long_opts = [ + 'chdir=', + 'file=', + 'fmt=', + 'format=', + 'help', + 'prefix=', + 'stage=', + 'tail=', + 'title=', + ] + + opts, args = getopt.getopt(argv[1:], short_opts, long_opts) + + for o, a in opts: + if o in ('-C', '--chdir'): + self.chdir = a + elif o in ('-f', '--file'): + self.config_file = a + elif o in ('--fmt', '--format'): + format = a + elif o in ('-?', '-h', '--help'): + self.do_help(['help', 'obj']) + sys.exit(0) + elif o in ('-p', '--prefix'): + self.prefix = a + elif o in ('--stage',): + if a not in self.stages: + sys.stderr.write('%s: obj: Unrecognized stage "%s".\n' % (self.name, a)) + sys.stderr.write('%s Type "%s help obj" for help.\n' % (self.name_spaces, self.name)) + sys.exit(1) + stage = a + elif o in ('-t', '--tail'): + tail = int(a) + elif o in ('--title',): + self.title = a + + if not args: + sys.stderr.write('%s: obj: Must specify an object name.\n' % self.name) + sys.stderr.write('%s Type "%s help obj" for help.\n' % (self.name_spaces, self.name)) + sys.exit(1) + + object_name = args.pop(0) + + if self.config_file: + with open(self.config_file, 'r') as f: + config = f.read() + HACK_for_exec(config, self.__dict__) + + if self.chdir: + os.chdir(self.chdir) + logfile_path = lambda x: os.path.join(self.chdir, x) + + if not args: + + pattern = '%s*.log' % self.prefix + args = self.args_to_files([pattern], tail) + + if not args: + if self.chdir: + directory = self.chdir + else: + directory = os.getcwd() + + sys.stderr.write('%s: obj: No arguments specified.\n' % self.name) + sys.stderr.write('%s No %s*.log files found in "%s".\n' % (self.name_spaces, self.prefix, directory)) + sys.stderr.write('%s Type "%s help obj" for help.\n' % (self.name_spaces, self.name)) + sys.exit(1) + + else: + + args = self.args_to_files(args, tail) + + cwd_ = os.getcwd() + os.sep + + if format == 'ascii': + + self.ascii_table(args, tuple(self.stages), self.get_object_counts, logfile_path, object_name) + + elif format == 'gnuplot': + + stage_index = 0 + for s in self.stages: + if stage == s: + break + stage_index = stage_index + 1 + + results = self.collect_results(args, self.get_object_counts, + object_name, stage_index) + + self.gnuplot_results(results) + + else: + + sys.stderr.write('%s: obj: Unknown format "%s".\n' % (self.name, format)) + sys.exit(1) + + return 0 + + # + + def help_run(self): + help = """\ + Usage: scons-time run [OPTIONS] [FILE ...] + + --chdir=DIR Name of unpacked directory for chdir + -f FILE, --file=FILE Read configuration from specified FILE + -h, --help Print this help and exit + -n, --no-exec No execute, just print command lines + --number=NUMBER Put output in files for run NUMBER + --outdir=OUTDIR Put output files in OUTDIR + -p STRING, --prefix=STRING Use STRING as log file/profile prefix + --python=PYTHON Time using the specified PYTHON + -q, --quiet Don't print command lines + --scons=SCONS Time using the specified SCONS + --svn=URL, --subversion=URL Use SCons from Subversion URL + -v, --verbose Display output of commands + """ + sys.stdout.write(self.outdent(help)) + sys.stdout.flush() + + def do_run(self, argv): + """ + """ + run_number_list = [None] + + short_opts = '?f:hnp:qs:v' + + long_opts = [ + 'file=', + 'help', + 'no-exec', + 'number=', + 'outdir=', + 'prefix=', + 'python=', + 'quiet', + 'scons=', + 'svn=', + 'subdir=', + 'subversion=', + 'verbose', + ] + + opts, args = getopt.getopt(argv[1:], short_opts, long_opts) + + for o, a in opts: + if o in ('-f', '--file'): + self.config_file = a + elif o in ('-?', '-h', '--help'): + self.do_help(['help', 'run']) + sys.exit(0) + elif o in ('-n', '--no-exec'): + self.execute = self._do_not_execute + elif o in ('--number',): + run_number_list = self.split_run_numbers(a) + elif o in ('--outdir',): + self.outdir = a + elif o in ('-p', '--prefix'): + self.prefix = a + elif o in ('--python',): + self.python = a + elif o in ('-q', '--quiet'): + self.display = self._do_not_display + elif o in ('-s', '--subdir'): + self.subdir = a + elif o in ('--scons',): + self.scons = a + elif o in ('--svn', '--subversion'): + self.subversion_url = a + elif o in ('-v', '--verbose'): + self.redirect = tee_to_file + self.verbose = True + self.svn_co_flag = '' + + if not args and not self.config_file: + sys.stderr.write('%s: run: No arguments or -f config file specified.\n' % self.name) + sys.stderr.write('%s Type "%s help run" for help.\n' % (self.name_spaces, self.name)) + sys.exit(1) + + if self.config_file: + with open(self.config_file, 'r') as f: + config = f.read() + exec(config, self.__dict__) + + if args: + self.archive_list = args + + archive_file_name = os.path.split(self.archive_list[0])[1] + + if not self.subdir: + self.subdir = self.archive_splitext(archive_file_name)[0] + + if not self.prefix: + self.prefix = self.archive_splitext(archive_file_name)[0] + + prepare = None + if self.subversion_url: + prepare = self.prep_subversion_run + + for run_number in run_number_list: + self.individual_run(run_number, self.archive_list, prepare) + + def split_run_numbers(self, s): + result = [] + for n in s.split(','): + try: + x, y = n.split('-') + except ValueError: + result.append(int(n)) + else: + result.extend(list(range(int(x), int(y)+1))) + return result + + def scons_path(self, dir): + return os.path.join(dir,'scripts', 'scons.py') + + def scons_lib_dir_path(self, dir): + return os.path.join(dir, 'src', 'engine') + + def prep_subversion_run(self, commands, removals): + self.svn_tmpdir = tempfile.mkdtemp(prefix=self.name + '-svn-') + removals.append((shutil.rmtree, 'rm -rf %%s', self.svn_tmpdir)) + + self.scons = self.scons_path(self.svn_tmpdir) + self.scons_lib_dir = self.scons_lib_dir_path(self.svn_tmpdir) + + commands.extend([ + '%(svn)s co %(svn_co_flag)s -r %(run_number)s %(subversion_url)s %(svn_tmpdir)s', + ]) + + def individual_run(self, run_number, archive_list, prepare=None): + """ + Performs an individual run of the default SCons invocations. + """ + + commands = [] + removals = [] + + if prepare: + prepare(commands, removals) + + save_scons = self.scons + save_scons_wrapper = self.scons_wrapper + save_scons_lib_dir = self.scons_lib_dir + + if self.outdir is None: + self.outdir = self.orig_cwd + elif not os.path.isabs(self.outdir): + self.outdir = os.path.join(self.orig_cwd, self.outdir) + + if self.scons is None: + self.scons = self.scons_path(self.orig_cwd) + + if self.scons_lib_dir is None: + self.scons_lib_dir = self.scons_lib_dir_path(self.orig_cwd) + + if self.scons_wrapper is None: + self.scons_wrapper = self.scons + + if not run_number: + run_number = self.find_next_run_number(self.outdir, self.prefix) + + self.run_number = str(run_number) + + self.prefix_run = self.prefix + '-%03d' % run_number + + if self.targets0 is None: + self.targets0 = self.startup_targets + if self.targets1 is None: + self.targets1 = self.targets + if self.targets2 is None: + self.targets2 = self.targets + + self.tmpdir = tempfile.mkdtemp(prefix=self.name + '-') + + commands.extend([ + (os.chdir, 'cd %%s', self.tmpdir), + ]) + + for archive in archive_list: + if not os.path.isabs(archive): + archive = os.path.join(self.orig_cwd, archive) + if os.path.isdir(archive): + dest = os.path.split(archive)[1] + commands.append((shutil.copytree, 'cp -r %%s %%s', archive, dest)) + else: + suffix = self.archive_splitext(archive)[1] + unpack_command = self.unpack_map.get(suffix) + if not unpack_command: + dest = os.path.split(archive)[1] + commands.append((shutil.copyfile, 'cp %%s %%s', archive, dest)) + else: + commands.append(unpack_command + (archive,)) + + commands.extend([ + (os.chdir, 'cd %%s', self.subdir), + ]) + + commands.extend(self.initial_commands) + + commands.extend([ + (lambda: read_tree('.'), + 'find * -type f | xargs cat > /dev/null'), + + (self.set_env, 'export %%s=%%s', + 'SCONS_LIB_DIR', self.scons_lib_dir), + + '%(python)s %(scons_wrapper)s --version', + ]) + + index = 0 + for run_command in self.run_commands: + setattr(self, 'prof%d' % index, self.profile_name(index)) + c = ( + self.log_execute, + self.log_display, + run_command, + self.logfile_name(index), + ) + commands.append(c) + index = index + 1 + + commands.extend([ + (os.chdir, 'cd %%s', self.orig_cwd), + ]) + + if not os.environ.get('PRESERVE'): + commands.extend(removals) + commands.append((shutil.rmtree, 'rm -rf %%s', self.tmpdir)) + + self.run_command_list(commands, self.__dict__) + + self.scons = save_scons + self.scons_lib_dir = save_scons_lib_dir + self.scons_wrapper = save_scons_wrapper + + # + + def help_time(self): + help = """\ + Usage: scons-time time [OPTIONS] FILE [...] + + -C DIR, --chdir=DIR Change to DIR before looking for files + -f FILE, --file=FILE Read configuration from specified FILE + --fmt=FORMAT, --format=FORMAT Print data in specified FORMAT + -h, --help Print this help and exit + -p STRING, --prefix=STRING Use STRING as log file/profile prefix + -t NUMBER, --tail=NUMBER Only report the last NUMBER files + --which=TIMER Plot timings for TIMER: total, + SConscripts, SCons, commands. + """ + sys.stdout.write(self.outdent(help)) + sys.stdout.flush() + + def do_time(self, argv): + + format = 'ascii' + logfile_path = lambda x: x + tail = None + which = 'total' + + short_opts = '?C:f:hp:t:' + + long_opts = [ + 'chdir=', + 'file=', + 'fmt=', + 'format=', + 'help', + 'prefix=', + 'tail=', + 'title=', + 'which=', + ] + + opts, args = getopt.getopt(argv[1:], short_opts, long_opts) + + for o, a in opts: + if o in ('-C', '--chdir'): + self.chdir = a + elif o in ('-f', '--file'): + self.config_file = a + elif o in ('--fmt', '--format'): + format = a + elif o in ('-?', '-h', '--help'): + self.do_help(['help', 'time']) + sys.exit(0) + elif o in ('-p', '--prefix'): + self.prefix = a + elif o in ('-t', '--tail'): + tail = int(a) + elif o in ('--title',): + self.title = a + elif o in ('--which',): + if a not in list(self.time_strings.keys()): + sys.stderr.write('%s: time: Unrecognized timer "%s".\n' % (self.name, a)) + sys.stderr.write('%s Type "%s help time" for help.\n' % (self.name_spaces, self.name)) + sys.exit(1) + which = a + + if self.config_file: + with open(self.config_file, 'r') as f: + config = f.read() + HACK_for_exec(config, self.__dict__) + + if self.chdir: + os.chdir(self.chdir) + logfile_path = lambda x: os.path.join(self.chdir, x) + + if not args: + + pattern = '%s*.log' % self.prefix + args = self.args_to_files([pattern], tail) + + if not args: + if self.chdir: + directory = self.chdir + else: + directory = os.getcwd() + + sys.stderr.write('%s: time: No arguments specified.\n' % self.name) + sys.stderr.write('%s No %s*.log files found in "%s".\n' % (self.name_spaces, self.prefix, directory)) + sys.stderr.write('%s Type "%s help time" for help.\n' % (self.name_spaces, self.name)) + sys.exit(1) + + else: + + args = self.args_to_files(args, tail) + + cwd_ = os.getcwd() + os.sep + + if format == 'ascii': + + columns = ("Total", "SConscripts", "SCons", "commands") + self.ascii_table(args, columns, self.get_debug_times, logfile_path) + + elif format == 'gnuplot': + + results = self.collect_results(args, self.get_debug_times, + self.time_strings[which]) + + self.gnuplot_results(results, fmt='%s %.6f') + + else: + + sys.stderr.write('%s: time: Unknown format "%s".\n' % (self.name, format)) + sys.exit(1) + +if __name__ == '__main__': + opts, args = getopt.getopt(sys.argv[1:], 'h?V', ['help', 'version']) + + ST = SConsTimer() + + for o, a in opts: + if o in ('-?', '-h', '--help'): + ST.do_help(['help']) + sys.exit(0) + elif o in ('-V', '--version'): + sys.stdout.write('scons-time version\n') + sys.exit(0) + + if not args: + sys.stderr.write('Type "%s help" for usage.\n' % ST.name) + sys.exit(1) + + ST.execute_subcommand(args) + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/runtest.py b/runtest.py index 65b2179..10d16ba 100755 --- a/runtest.py +++ b/runtest.py @@ -478,90 +478,49 @@ else: Test = SystemExecutor # --- start processing --- -if package: - - dirs = { - 'deb' : 'usr', - 'local-tar-gz' : None, - 'local-zip' : None, - 'rpm' : 'usr', - 'src-tar-gz' : '', - 'src-zip' : '', - 'tar-gz' : '', - 'zip' : '', - } - - # The hard-coded "python2.1" here is the library directory - # name on Debian systems, not an executable, so it's all right. - lib = { - 'deb' : os.path.join('python2.1', 'site-packages') - } - - if package not in dirs: - sys.stderr.write("Unknown package '%s'\n" % package) - sys.exit(2) - - test_dir = os.path.join(builddir, 'test-%s' % package) - - if dirs[package] is None: - scons_script_dir = test_dir - globs = glob.glob(os.path.join(test_dir, 'scons-local-*')) - if not globs: - sys.stderr.write("No `scons-local-*' dir in `%s'\n" % test_dir) - sys.exit(2) - scons_lib_dir = None - pythonpath_dir = globs[len(globs)-1] - elif sys.platform == 'win32': - scons_script_dir = os.path.join(test_dir, dirs[package], 'Scripts') - scons_lib_dir = os.path.join(test_dir, dirs[package]) - pythonpath_dir = scons_lib_dir - else: - scons_script_dir = os.path.join(test_dir, dirs[package], 'bin') - sconslib = lib.get(package, 'scons') - scons_lib_dir = os.path.join(test_dir, dirs[package], 'lib', sconslib) - pythonpath_dir = scons_lib_dir - scons_runtest_dir = builddir +sd = None +tools_dir = None +ld = None + +if not baseline or baseline == '.': + base = cwd +elif baseline == '-': + url = None + with os.popen("svn info 2>&1", "r") as p: + svn_info = p.read() + match = re.search(r'URL: (.*)', svn_info) + if match: + url = match.group(1) + if not url: + sys.stderr.write('runtest.py: could not find a URL:\n') + sys.stderr.write(svn_info) + sys.exit(1) + import tempfile + base = tempfile.mkdtemp(prefix='runtest-tmp-') + + command = 'cd %s && svn co -q %s' % (base, url) + base = os.path.join(base, os.path.split(url)[1]) + if printcommand: + print(command) + if execute_tests: + os.system(command) else: - sd = None - ld = None - - if not baseline or baseline == '.': - base = cwd - elif baseline == '-': - url = None - with os.popen("svn info 2>&1", "r") as p: - svn_info = p.read() - match = re.search(r'URL: (.*)', svn_info) - if match: - url = match.group(1) - if not url: - sys.stderr.write('runtest.py: could not find a URL:\n') - sys.stderr.write(svn_info) - sys.exit(1) - base = tempfile.mkdtemp(prefix='runtest-tmp-') - - command = 'cd %s && svn co -q %s' % (base, url) - - base = os.path.join(base, os.path.split(url)[1]) - if printcommand: - print(command) - if execute_tests: - os.system(command) - else: - base = baseline + base = baseline - scons_runtest_dir = base +scons_runtest_dir = base - if not external: - scons_script_dir = sd or os.path.join(base, 'scripts') - scons_lib_dir = ld or os.path.join(base, 'src', 'engine') - else: - scons_script_dir = sd or '' - scons_lib_dir = ld or '' +if not external: + scons_script_dir = sd or os.path.join(base, 'scripts') + scons_tools_dir = tools_dir or os.path.join(base, 'bin') + scons_lib_dir = ld or os.path.join(base, 'src', 'engine') +else: + scons_script_dir = sd or '' + scons_tools_dir = tools_dir or '' + scons_lib_dir = ld or '' - pythonpath_dir = scons_lib_dir +pythonpath_dir = scons_lib_dir if scons: # Let the version of SCons that the -x option pointed to find @@ -581,6 +540,7 @@ if external: os.environ['SCONS_RUNTEST_DIR'] = scons_runtest_dir os.environ['SCONS_SCRIPT_DIR'] = scons_script_dir +os.environ['SCONS_TOOLS_DIR'] = scons_tools_dir os.environ['SCONS_CWD'] = cwd os.environ['SCONS_VERSION'] = version diff --git a/scripts/scons-time.py b/scripts/scons-time.py deleted file mode 100644 index 6494349..0000000 --- a/scripts/scons-time.py +++ /dev/null @@ -1,1480 +0,0 @@ -#!/usr/bin/env python -# -# scons-time - run SCons timings and collect statistics -# -# A script for running a configuration through SCons with a standard -# set of invocations to collect timing and memory statistics and to -# capture the results in a consistent set of output files for display -# and analysis. -# - -# -# __COPYRIGHT__ -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - -import getopt -import glob -import os -import re -import shutil -import sys -import tempfile -import time -import subprocess - -def HACK_for_exec(cmd, *args): - """ - For some reason, Python won't allow an exec() within a function - that also declares an internal function (including lambda functions). - This function is a hack that calls exec() in a function with no - internal functions. - """ - if not args: exec(cmd) - elif len(args) == 1: exec(cmd, args[0]) - else: exec(cmd, args[0], args[1]) - -class Plotter(object): - def increment_size(self, largest): - """ - Return the size of each horizontal increment line for a specified - maximum value. This returns a value that will provide somewhere - between 5 and 9 horizontal lines on the graph, on some set of - boundaries that are multiples of 10/100/1000/etc. - """ - i = largest // 5 - if not i: - return largest - multiplier = 1 - while i >= 10: - i = i // 10 - multiplier = multiplier * 10 - return i * multiplier - - def max_graph_value(self, largest): - # Round up to next integer. - largest = int(largest) + 1 - increment = self.increment_size(largest) - return ((largest + increment - 1) // increment) * increment - -class Line(object): - def __init__(self, points, type, title, label, comment, fmt="%s %s"): - self.points = points - self.type = type - self.title = title - self.label = label - self.comment = comment - self.fmt = fmt - - def print_label(self, inx, x, y): - if self.label: - print('set label %s "%s" at %0.1f,%0.1f right' % (inx, self.label, x, y)) - - def plot_string(self): - if self.title: - title_string = 'title "%s"' % self.title - else: - title_string = 'notitle' - return "'-' %s with lines lt %s" % (title_string, self.type) - - def print_points(self, fmt=None): - if fmt is None: - fmt = self.fmt - if self.comment: - print('# %s' % self.comment) - for x, y in self.points: - # If y is None, it usually represents some kind of break - # in the line's index number. We might want to represent - # this some way rather than just drawing the line straight - # between the two points on either side. - if y is not None: - print(fmt % (x, y)) - print('e') - - def get_x_values(self): - return [ p[0] for p in self.points ] - - def get_y_values(self): - return [ p[1] for p in self.points ] - -class Gnuplotter(Plotter): - - def __init__(self, title, key_location): - self.lines = [] - self.title = title - self.key_location = key_location - - def line(self, points, type, title=None, label=None, comment=None, fmt='%s %s'): - if points: - line = Line(points, type, title, label, comment, fmt) - self.lines.append(line) - - def plot_string(self, line): - return line.plot_string() - - def vertical_bar(self, x, type, label, comment): - if self.get_min_x() <= x <= self.get_max_x(): - points = [(x, 0), (x, self.max_graph_value(self.get_max_y()))] - self.line(points, type, label, comment) - - def get_all_x_values(self): - result = [] - for line in self.lines: - result.extend(line.get_x_values()) - return [r for r in result if r is not None] - - def get_all_y_values(self): - result = [] - for line in self.lines: - result.extend(line.get_y_values()) - return [r for r in result if r is not None] - - def get_min_x(self): - try: - return self.min_x - except AttributeError: - try: - self.min_x = min(self.get_all_x_values()) - except ValueError: - self.min_x = 0 - return self.min_x - - def get_max_x(self): - try: - return self.max_x - except AttributeError: - try: - self.max_x = max(self.get_all_x_values()) - except ValueError: - self.max_x = 0 - return self.max_x - - def get_min_y(self): - try: - return self.min_y - except AttributeError: - try: - self.min_y = min(self.get_all_y_values()) - except ValueError: - self.min_y = 0 - return self.min_y - - def get_max_y(self): - try: - return self.max_y - except AttributeError: - try: - self.max_y = max(self.get_all_y_values()) - except ValueError: - self.max_y = 0 - return self.max_y - - def draw(self): - - if not self.lines: - return - - if self.title: - print('set title "%s"' % self.title) - print('set key %s' % self.key_location) - - min_y = self.get_min_y() - max_y = self.max_graph_value(self.get_max_y()) - incr = (max_y - min_y) / 10.0 - start = min_y + (max_y / 2.0) + (2.0 * incr) - position = [ start - (i * incr) for i in range(5) ] - - inx = 1 - for line in self.lines: - line.print_label(inx, line.points[0][0]-1, - position[(inx-1) % len(position)]) - inx += 1 - - plot_strings = [ self.plot_string(l) for l in self.lines ] - print('plot ' + ', \\\n '.join(plot_strings)) - - for line in self.lines: - line.print_points() - - - -def untar(fname): - import tarfile - tar = tarfile.open(name=fname, mode='r') - for tarinfo in tar: - tar.extract(tarinfo) - tar.close() - -def unzip(fname): - import zipfile - zf = zipfile.ZipFile(fname, 'r') - for name in zf.namelist(): - dir = os.path.dirname(name) - try: - os.makedirs(dir) - except: - pass - with open(name, 'wb') as f: - f.write(zf.read(name)) - -def read_tree(dir): - for dirpath, dirnames, filenames in os.walk(dir): - for fn in filenames: - fn = os.path.join(dirpath, fn) - if os.path.isfile(fn): - with open(fn, 'rb') as f: - f.read() - -def redirect_to_file(command, log): - return '%s > %s 2>&1' % (command, log) - -def tee_to_file(command, log): - return '%s 2>&1 | tee %s' % (command, log) - - - -class SConsTimer(object): - """ - Usage: scons-time SUBCOMMAND [ARGUMENTS] - Type "scons-time help SUBCOMMAND" for help on a specific subcommand. - - Available subcommands: - func Extract test-run data for a function - help Provides help - mem Extract --debug=memory data from test runs - obj Extract --debug=count data from test runs - time Extract --debug=time data from test runs - run Runs a test configuration - """ - - name = 'scons-time' - name_spaces = ' '*len(name) - - def makedict(**kw): - return kw - - default_settings = makedict( - chdir = None, - config_file = None, - initial_commands = [], - key_location = 'bottom left', - orig_cwd = os.getcwd(), - outdir = None, - prefix = '', - python = '"%s"' % sys.executable, - redirect = redirect_to_file, - scons = None, - scons_flags = '--debug=count --debug=memory --debug=time --debug=memoizer', - scons_lib_dir = None, - scons_wrapper = None, - startup_targets = '--help', - subdir = None, - subversion_url = None, - svn = 'svn', - svn_co_flag = '-q', - tar = 'tar', - targets = '', - targets0 = None, - targets1 = None, - targets2 = None, - title = None, - unzip = 'unzip', - verbose = False, - vertical_bars = [], - - unpack_map = { - '.tar.gz' : (untar, '%(tar)s xzf %%s'), - '.tgz' : (untar, '%(tar)s xzf %%s'), - '.tar' : (untar, '%(tar)s xf %%s'), - '.zip' : (unzip, '%(unzip)s %%s'), - }, - ) - - run_titles = [ - 'Startup', - 'Full build', - 'Up-to-date build', - ] - - run_commands = [ - '%(python)s %(scons_wrapper)s %(scons_flags)s --profile=%(prof0)s %(targets0)s', - '%(python)s %(scons_wrapper)s %(scons_flags)s --profile=%(prof1)s %(targets1)s', - '%(python)s %(scons_wrapper)s %(scons_flags)s --profile=%(prof2)s %(targets2)s', - ] - - stages = [ - 'pre-read', - 'post-read', - 'pre-build', - 'post-build', - ] - - stage_strings = { - 'pre-read' : 'Memory before reading SConscript files:', - 'post-read' : 'Memory after reading SConscript files:', - 'pre-build' : 'Memory before building targets:', - 'post-build' : 'Memory after building targets:', - } - - memory_string_all = 'Memory ' - - default_stage = stages[-1] - - time_strings = { - 'total' : 'Total build time', - 'SConscripts' : 'Total SConscript file execution time', - 'SCons' : 'Total SCons execution time', - 'commands' : 'Total command execution time', - } - - time_string_all = 'Total .* time' - - # - - def __init__(self): - self.__dict__.update(self.default_settings) - - # Functions for displaying and executing commands. - - def subst(self, x, dictionary): - try: - return x % dictionary - except TypeError: - # x isn't a string (it's probably a Python function), - # so just return it. - return x - - def subst_variables(self, command, dictionary): - """ - Substitutes (via the format operator) the values in the specified - dictionary into the specified command. - - The command can be an (action, string) tuple. In all cases, we - perform substitution on strings and don't worry if something isn't - a string. (It's probably a Python function to be executed.) - """ - try: - command + '' - except TypeError: - action = command[0] - string = command[1] - args = command[2:] - else: - action = command - string = action - args = (()) - action = self.subst(action, dictionary) - string = self.subst(string, dictionary) - return (action, string, args) - - def _do_not_display(self, msg, *args): - pass - - def display(self, msg, *args): - """ - Displays the specified message. - - Each message is prepended with a standard prefix of our name - plus the time. - """ - if callable(msg): - msg = msg(*args) - else: - msg = msg % args - if msg is None: - return - fmt = '%s[%s]: %s\n' - sys.stdout.write(fmt % (self.name, time.strftime('%H:%M:%S'), msg)) - - def _do_not_execute(self, action, *args): - pass - - def execute(self, action, *args): - """ - Executes the specified action. - - The action is called if it's a callable Python function, and - otherwise passed to os.system(). - """ - if callable(action): - action(*args) - else: - os.system(action % args) - - def run_command_list(self, commands, dict): - """ - Executes a list of commands, substituting values from the - specified dictionary. - """ - commands = [ self.subst_variables(c, dict) for c in commands ] - for action, string, args in commands: - self.display(string, *args) - sys.stdout.flush() - status = self.execute(action, *args) - if status: - sys.exit(status) - - def log_display(self, command, log): - command = self.subst(command, self.__dict__) - if log: - command = self.redirect(command, log) - return command - - def log_execute(self, command, log): - command = self.subst(command, self.__dict__) - p = os.popen(command) - output = p.read() - p.close() - #TODO: convert to subrocess, os.popen is obsolete. This didn't work: - #process = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True) - #output = process.stdout.read() - #process.stdout.close() - #process.wait() - if self.verbose: - sys.stdout.write(output) - # TODO: Figure out - # Not sure we need to write binary here - with open(log, 'w') as f: - f.write(str(output)) - - def archive_splitext(self, path): - """ - Splits an archive name into a filename base and extension. - - This is like os.path.splitext() (which it calls) except that it - also looks for '.tar.gz' and treats it as an atomic extensions. - """ - if path.endswith('.tar.gz'): - return path[:-7], path[-7:] - else: - return os.path.splitext(path) - - def args_to_files(self, args, tail=None): - """ - Takes a list of arguments, expands any glob patterns, and - returns the last "tail" files from the list. - """ - files = [] - for a in args: - files.extend(sorted(glob.glob(a))) - - if tail: - files = files[-tail:] - - return files - - def ascii_table(self, files, columns, - line_function, file_function=lambda x: x, - *args, **kw): - - header_fmt = ' '.join(['%12s'] * len(columns)) - line_fmt = header_fmt + ' %s' - - print(header_fmt % columns) - - for file in files: - t = line_function(file, *args, **kw) - if t is None: - t = [] - diff = len(columns) - len(t) - if diff > 0: - t += [''] * diff - t.append(file_function(file)) - print(line_fmt % tuple(t)) - - def collect_results(self, files, function, *args, **kw): - results = {} - - for file in files: - base = os.path.splitext(file)[0] - run, index = base.split('-')[-2:] - - run = int(run) - index = int(index) - - value = function(file, *args, **kw) - - try: - r = results[index] - except KeyError: - r = [] - results[index] = r - r.append((run, value)) - - return results - - def doc_to_help(self, obj): - """ - Translates an object's __doc__ string into help text. - - This strips a consistent number of spaces from each line in the - help text, essentially "outdenting" the text to the left-most - column. - """ - doc = obj.__doc__ - if doc is None: - return '' - return self.outdent(doc) - - def find_next_run_number(self, dir, prefix): - """ - Returns the next run number in a directory for the specified prefix. - - Examines the contents the specified directory for files with the - specified prefix, extracts the run numbers from each file name, - and returns the next run number after the largest it finds. - """ - x = re.compile(re.escape(prefix) + '-([0-9]+).*') - matches = [x.match(e) for e in os.listdir(dir)] - matches = [_f for _f in matches if _f] - if not matches: - return 0 - run_numbers = [int(m.group(1)) for m in matches] - return int(max(run_numbers)) + 1 - - def gnuplot_results(self, results, fmt='%s %.3f'): - """ - Prints out a set of results in Gnuplot format. - """ - gp = Gnuplotter(self.title, self.key_location) - - for i in sorted(results.keys()): - try: - t = self.run_titles[i] - except IndexError: - t = '??? %s ???' % i - results[i].sort() - gp.line(results[i], i+1, t, None, t, fmt=fmt) - - for bar_tuple in self.vertical_bars: - try: - x, type, label, comment = bar_tuple - except ValueError: - x, type, label = bar_tuple - comment = label - gp.vertical_bar(x, type, label, comment) - - gp.draw() - - def logfile_name(self, invocation): - """ - Returns the absolute path of a log file for the specificed - invocation number. - """ - name = self.prefix_run + '-%d.log' % invocation - return os.path.join(self.outdir, name) - - def outdent(self, s): - """ - Strip as many spaces from each line as are found at the beginning - of the first line in the list. - """ - lines = s.split('\n') - if lines[0] == '': - lines = lines[1:] - spaces = re.match(' *', lines[0]).group(0) - def strip_initial_spaces(l, s=spaces): - if l.startswith(spaces): - l = l[len(spaces):] - return l - return '\n'.join([ strip_initial_spaces(l) for l in lines ]) + '\n' - - def profile_name(self, invocation): - """ - Returns the absolute path of a profile file for the specified - invocation number. - """ - name = self.prefix_run + '-%d.prof' % invocation - return os.path.join(self.outdir, name) - - def set_env(self, key, value): - os.environ[key] = value - - # - - def get_debug_times(self, file, time_string=None): - """ - Fetch times from the --debug=time strings in the specified file. - """ - if time_string is None: - search_string = self.time_string_all - else: - search_string = time_string - with open(file) as f: - contents = f.read() - if not contents: - sys.stderr.write('file %s has no contents!\n' % repr(file)) - return None - result = re.findall(r'%s: ([\d.]*)' % search_string, contents)[-4:] - result = [ float(r) for r in result ] - if time_string is not None: - try: - result = result[0] - except IndexError: - sys.stderr.write('file %s has no results!\n' % repr(file)) - return None - return result - - def get_function_profile(self, file, function): - """ - Returns the file, line number, function name, and cumulative time. - """ - try: - import pstats - except ImportError as e: - sys.stderr.write('%s: func: %s\n' % (self.name, e)) - sys.stderr.write('%s This version of Python is missing the profiler.\n' % self.name_spaces) - sys.stderr.write('%s Cannot use the "func" subcommand.\n' % self.name_spaces) - sys.exit(1) - statistics = pstats.Stats(file).stats - matches = [ e for e in statistics.items() if e[0][2] == function ] - r = matches[0] - return r[0][0], r[0][1], r[0][2], r[1][3] - - def get_function_time(self, file, function): - """ - Returns just the cumulative time for the specified function. - """ - return self.get_function_profile(file, function)[3] - - def get_memory(self, file, memory_string=None): - """ - Returns a list of integers of the amount of memory used. The - default behavior is to return all the stages. - """ - if memory_string is None: - search_string = self.memory_string_all - else: - search_string = memory_string - with open(file) as f: - lines = f.readlines() - lines = [ l for l in lines if l.startswith(search_string) ][-4:] - result = [ int(l.split()[-1]) for l in lines[-4:] ] - if len(result) == 1: - result = result[0] - return result - - def get_object_counts(self, file, object_name, index=None): - """ - Returns the counts of the specified object_name. - """ - object_string = ' ' + object_name + '\n' - with open(file) as f: - lines = f.readlines() - line = [ l for l in lines if l.endswith(object_string) ][0] - result = [ int(field) for field in line.split()[:4] ] - if index is not None: - result = result[index] - return result - - - command_alias = {} - - def execute_subcommand(self, argv): - """ - Executes the do_*() function for the specified subcommand (argv[0]). - """ - if not argv: - return - cmdName = self.command_alias.get(argv[0], argv[0]) - try: - func = getattr(self, 'do_' + cmdName) - except AttributeError: - return self.default(argv) - try: - return func(argv) - except TypeError as e: - sys.stderr.write("%s %s: %s\n" % (self.name, cmdName, e)) - import traceback - traceback.print_exc(file=sys.stderr) - sys.stderr.write("Try '%s help %s'\n" % (self.name, cmdName)) - - def default(self, argv): - """ - The default behavior for an unknown subcommand. Prints an - error message and exits. - """ - sys.stderr.write('%s: Unknown subcommand "%s".\n' % (self.name, argv[0])) - sys.stderr.write('Type "%s help" for usage.\n' % self.name) - sys.exit(1) - - # - - def do_help(self, argv): - """ - """ - if argv[1:]: - for arg in argv[1:]: - try: - func = getattr(self, 'do_' + arg) - except AttributeError: - sys.stderr.write('%s: No help for "%s"\n' % (self.name, arg)) - else: - try: - help = getattr(self, 'help_' + arg) - except AttributeError: - sys.stdout.write(self.doc_to_help(func)) - sys.stdout.flush() - else: - help() - else: - doc = self.doc_to_help(self.__class__) - if doc: - sys.stdout.write(doc) - sys.stdout.flush() - return None - - # - - def help_func(self): - help = """\ - Usage: scons-time func [OPTIONS] FILE [...] - - -C DIR, --chdir=DIR Change to DIR before looking for files - -f FILE, --file=FILE Read configuration from specified FILE - --fmt=FORMAT, --format=FORMAT Print data in specified FORMAT - --func=NAME, --function=NAME Report time for function NAME - -h, --help Print this help and exit - -p STRING, --prefix=STRING Use STRING as log file/profile prefix - -t NUMBER, --tail=NUMBER Only report the last NUMBER files - --title=TITLE Specify the output plot TITLE - """ - sys.stdout.write(self.outdent(help)) - sys.stdout.flush() - - def do_func(self, argv): - """ - """ - format = 'ascii' - function_name = '_main' - tail = None - - short_opts = '?C:f:hp:t:' - - long_opts = [ - 'chdir=', - 'file=', - 'fmt=', - 'format=', - 'func=', - 'function=', - 'help', - 'prefix=', - 'tail=', - 'title=', - ] - - opts, args = getopt.getopt(argv[1:], short_opts, long_opts) - - for o, a in opts: - if o in ('-C', '--chdir'): - self.chdir = a - elif o in ('-f', '--file'): - self.config_file = a - elif o in ('--fmt', '--format'): - format = a - elif o in ('--func', '--function'): - function_name = a - elif o in ('-?', '-h', '--help'): - self.do_help(['help', 'func']) - sys.exit(0) - elif o in ('--max',): - max_time = int(a) - elif o in ('-p', '--prefix'): - self.prefix = a - elif o in ('-t', '--tail'): - tail = int(a) - elif o in ('--title',): - self.title = a - - if self.config_file: - with open(self.config_file, 'r') as f: - config = f.read() - exec(config, self.__dict__) - - if self.chdir: - os.chdir(self.chdir) - - if not args: - - pattern = '%s*.prof' % self.prefix - args = self.args_to_files([pattern], tail) - - if not args: - if self.chdir: - directory = self.chdir - else: - directory = os.getcwd() - - sys.stderr.write('%s: func: No arguments specified.\n' % self.name) - sys.stderr.write('%s No %s*.prof files found in "%s".\n' % (self.name_spaces, self.prefix, directory)) - sys.stderr.write('%s Type "%s help func" for help.\n' % (self.name_spaces, self.name)) - sys.exit(1) - - else: - - args = self.args_to_files(args, tail) - - cwd_ = os.getcwd() + os.sep - - if format == 'ascii': - - for file in args: - try: - f, line, func, time = \ - self.get_function_profile(file, function_name) - except ValueError as e: - sys.stderr.write("%s: func: %s: %s\n" % - (self.name, file, e)) - else: - if f.startswith(cwd_): - f = f[len(cwd_):] - print("%.3f %s:%d(%s)" % (time, f, line, func)) - - elif format == 'gnuplot': - - results = self.collect_results(args, self.get_function_time, - function_name) - - self.gnuplot_results(results) - - else: - - sys.stderr.write('%s: func: Unknown format "%s".\n' % (self.name, format)) - sys.exit(1) - - # - - def help_mem(self): - help = """\ - Usage: scons-time mem [OPTIONS] FILE [...] - - -C DIR, --chdir=DIR Change to DIR before looking for files - -f FILE, --file=FILE Read configuration from specified FILE - --fmt=FORMAT, --format=FORMAT Print data in specified FORMAT - -h, --help Print this help and exit - -p STRING, --prefix=STRING Use STRING as log file/profile prefix - --stage=STAGE Plot memory at the specified stage: - pre-read, post-read, pre-build, - post-build (default: post-build) - -t NUMBER, --tail=NUMBER Only report the last NUMBER files - --title=TITLE Specify the output plot TITLE - """ - sys.stdout.write(self.outdent(help)) - sys.stdout.flush() - - def do_mem(self, argv): - - format = 'ascii' - logfile_path = lambda x: x - stage = self.default_stage - tail = None - - short_opts = '?C:f:hp:t:' - - long_opts = [ - 'chdir=', - 'file=', - 'fmt=', - 'format=', - 'help', - 'prefix=', - 'stage=', - 'tail=', - 'title=', - ] - - opts, args = getopt.getopt(argv[1:], short_opts, long_opts) - - for o, a in opts: - if o in ('-C', '--chdir'): - self.chdir = a - elif o in ('-f', '--file'): - self.config_file = a - elif o in ('--fmt', '--format'): - format = a - elif o in ('-?', '-h', '--help'): - self.do_help(['help', 'mem']) - sys.exit(0) - elif o in ('-p', '--prefix'): - self.prefix = a - elif o in ('--stage',): - if a not in self.stages: - sys.stderr.write('%s: mem: Unrecognized stage "%s".\n' % (self.name, a)) - sys.exit(1) - stage = a - elif o in ('-t', '--tail'): - tail = int(a) - elif o in ('--title',): - self.title = a - - if self.config_file: - with open(self.config_file, 'r') as f: - config = f.read() - HACK_for_exec(config, self.__dict__) - - if self.chdir: - os.chdir(self.chdir) - logfile_path = lambda x: os.path.join(self.chdir, x) - - if not args: - - pattern = '%s*.log' % self.prefix - args = self.args_to_files([pattern], tail) - - if not args: - if self.chdir: - directory = self.chdir - else: - directory = os.getcwd() - - sys.stderr.write('%s: mem: No arguments specified.\n' % self.name) - sys.stderr.write('%s No %s*.log files found in "%s".\n' % (self.name_spaces, self.prefix, directory)) - sys.stderr.write('%s Type "%s help mem" for help.\n' % (self.name_spaces, self.name)) - sys.exit(1) - - else: - - args = self.args_to_files(args, tail) - - cwd_ = os.getcwd() + os.sep - - if format == 'ascii': - - self.ascii_table(args, tuple(self.stages), self.get_memory, logfile_path) - - elif format == 'gnuplot': - - results = self.collect_results(args, self.get_memory, - self.stage_strings[stage]) - - self.gnuplot_results(results) - - else: - - sys.stderr.write('%s: mem: Unknown format "%s".\n' % (self.name, format)) - sys.exit(1) - - return 0 - - # - - def help_obj(self): - help = """\ - Usage: scons-time obj [OPTIONS] OBJECT FILE [...] - - -C DIR, --chdir=DIR Change to DIR before looking for files - -f FILE, --file=FILE Read configuration from specified FILE - --fmt=FORMAT, --format=FORMAT Print data in specified FORMAT - -h, --help Print this help and exit - -p STRING, --prefix=STRING Use STRING as log file/profile prefix - --stage=STAGE Plot memory at the specified stage: - pre-read, post-read, pre-build, - post-build (default: post-build) - -t NUMBER, --tail=NUMBER Only report the last NUMBER files - --title=TITLE Specify the output plot TITLE - """ - sys.stdout.write(self.outdent(help)) - sys.stdout.flush() - - def do_obj(self, argv): - - format = 'ascii' - logfile_path = lambda x: x - stage = self.default_stage - tail = None - - short_opts = '?C:f:hp:t:' - - long_opts = [ - 'chdir=', - 'file=', - 'fmt=', - 'format=', - 'help', - 'prefix=', - 'stage=', - 'tail=', - 'title=', - ] - - opts, args = getopt.getopt(argv[1:], short_opts, long_opts) - - for o, a in opts: - if o in ('-C', '--chdir'): - self.chdir = a - elif o in ('-f', '--file'): - self.config_file = a - elif o in ('--fmt', '--format'): - format = a - elif o in ('-?', '-h', '--help'): - self.do_help(['help', 'obj']) - sys.exit(0) - elif o in ('-p', '--prefix'): - self.prefix = a - elif o in ('--stage',): - if a not in self.stages: - sys.stderr.write('%s: obj: Unrecognized stage "%s".\n' % (self.name, a)) - sys.stderr.write('%s Type "%s help obj" for help.\n' % (self.name_spaces, self.name)) - sys.exit(1) - stage = a - elif o in ('-t', '--tail'): - tail = int(a) - elif o in ('--title',): - self.title = a - - if not args: - sys.stderr.write('%s: obj: Must specify an object name.\n' % self.name) - sys.stderr.write('%s Type "%s help obj" for help.\n' % (self.name_spaces, self.name)) - sys.exit(1) - - object_name = args.pop(0) - - if self.config_file: - with open(self.config_file, 'r') as f: - config = f.read() - HACK_for_exec(config, self.__dict__) - - if self.chdir: - os.chdir(self.chdir) - logfile_path = lambda x: os.path.join(self.chdir, x) - - if not args: - - pattern = '%s*.log' % self.prefix - args = self.args_to_files([pattern], tail) - - if not args: - if self.chdir: - directory = self.chdir - else: - directory = os.getcwd() - - sys.stderr.write('%s: obj: No arguments specified.\n' % self.name) - sys.stderr.write('%s No %s*.log files found in "%s".\n' % (self.name_spaces, self.prefix, directory)) - sys.stderr.write('%s Type "%s help obj" for help.\n' % (self.name_spaces, self.name)) - sys.exit(1) - - else: - - args = self.args_to_files(args, tail) - - cwd_ = os.getcwd() + os.sep - - if format == 'ascii': - - self.ascii_table(args, tuple(self.stages), self.get_object_counts, logfile_path, object_name) - - elif format == 'gnuplot': - - stage_index = 0 - for s in self.stages: - if stage == s: - break - stage_index = stage_index + 1 - - results = self.collect_results(args, self.get_object_counts, - object_name, stage_index) - - self.gnuplot_results(results) - - else: - - sys.stderr.write('%s: obj: Unknown format "%s".\n' % (self.name, format)) - sys.exit(1) - - return 0 - - # - - def help_run(self): - help = """\ - Usage: scons-time run [OPTIONS] [FILE ...] - - --chdir=DIR Name of unpacked directory for chdir - -f FILE, --file=FILE Read configuration from specified FILE - -h, --help Print this help and exit - -n, --no-exec No execute, just print command lines - --number=NUMBER Put output in files for run NUMBER - --outdir=OUTDIR Put output files in OUTDIR - -p STRING, --prefix=STRING Use STRING as log file/profile prefix - --python=PYTHON Time using the specified PYTHON - -q, --quiet Don't print command lines - --scons=SCONS Time using the specified SCONS - --svn=URL, --subversion=URL Use SCons from Subversion URL - -v, --verbose Display output of commands - """ - sys.stdout.write(self.outdent(help)) - sys.stdout.flush() - - def do_run(self, argv): - """ - """ - run_number_list = [None] - - short_opts = '?f:hnp:qs:v' - - long_opts = [ - 'file=', - 'help', - 'no-exec', - 'number=', - 'outdir=', - 'prefix=', - 'python=', - 'quiet', - 'scons=', - 'svn=', - 'subdir=', - 'subversion=', - 'verbose', - ] - - opts, args = getopt.getopt(argv[1:], short_opts, long_opts) - - for o, a in opts: - if o in ('-f', '--file'): - self.config_file = a - elif o in ('-?', '-h', '--help'): - self.do_help(['help', 'run']) - sys.exit(0) - elif o in ('-n', '--no-exec'): - self.execute = self._do_not_execute - elif o in ('--number',): - run_number_list = self.split_run_numbers(a) - elif o in ('--outdir',): - self.outdir = a - elif o in ('-p', '--prefix'): - self.prefix = a - elif o in ('--python',): - self.python = a - elif o in ('-q', '--quiet'): - self.display = self._do_not_display - elif o in ('-s', '--subdir'): - self.subdir = a - elif o in ('--scons',): - self.scons = a - elif o in ('--svn', '--subversion'): - self.subversion_url = a - elif o in ('-v', '--verbose'): - self.redirect = tee_to_file - self.verbose = True - self.svn_co_flag = '' - - if not args and not self.config_file: - sys.stderr.write('%s: run: No arguments or -f config file specified.\n' % self.name) - sys.stderr.write('%s Type "%s help run" for help.\n' % (self.name_spaces, self.name)) - sys.exit(1) - - if self.config_file: - with open(self.config_file, 'r') as f: - config = f.read() - exec(config, self.__dict__) - - if args: - self.archive_list = args - - archive_file_name = os.path.split(self.archive_list[0])[1] - - if not self.subdir: - self.subdir = self.archive_splitext(archive_file_name)[0] - - if not self.prefix: - self.prefix = self.archive_splitext(archive_file_name)[0] - - prepare = None - if self.subversion_url: - prepare = self.prep_subversion_run - - for run_number in run_number_list: - self.individual_run(run_number, self.archive_list, prepare) - - def split_run_numbers(self, s): - result = [] - for n in s.split(','): - try: - x, y = n.split('-') - except ValueError: - result.append(int(n)) - else: - result.extend(list(range(int(x), int(y)+1))) - return result - - def scons_path(self, dir): - return os.path.join(dir,'scripts', 'scons.py') - - def scons_lib_dir_path(self, dir): - return os.path.join(dir, 'src', 'engine') - - def prep_subversion_run(self, commands, removals): - self.svn_tmpdir = tempfile.mkdtemp(prefix=self.name + '-svn-') - removals.append((shutil.rmtree, 'rm -rf %%s', self.svn_tmpdir)) - - self.scons = self.scons_path(self.svn_tmpdir) - self.scons_lib_dir = self.scons_lib_dir_path(self.svn_tmpdir) - - commands.extend([ - '%(svn)s co %(svn_co_flag)s -r %(run_number)s %(subversion_url)s %(svn_tmpdir)s', - ]) - - def individual_run(self, run_number, archive_list, prepare=None): - """ - Performs an individual run of the default SCons invocations. - """ - - commands = [] - removals = [] - - if prepare: - prepare(commands, removals) - - save_scons = self.scons - save_scons_wrapper = self.scons_wrapper - save_scons_lib_dir = self.scons_lib_dir - - if self.outdir is None: - self.outdir = self.orig_cwd - elif not os.path.isabs(self.outdir): - self.outdir = os.path.join(self.orig_cwd, self.outdir) - - if self.scons is None: - self.scons = self.scons_path(self.orig_cwd) - - if self.scons_lib_dir is None: - self.scons_lib_dir = self.scons_lib_dir_path(self.orig_cwd) - - if self.scons_wrapper is None: - self.scons_wrapper = self.scons - - if not run_number: - run_number = self.find_next_run_number(self.outdir, self.prefix) - - self.run_number = str(run_number) - - self.prefix_run = self.prefix + '-%03d' % run_number - - if self.targets0 is None: - self.targets0 = self.startup_targets - if self.targets1 is None: - self.targets1 = self.targets - if self.targets2 is None: - self.targets2 = self.targets - - self.tmpdir = tempfile.mkdtemp(prefix=self.name + '-') - - commands.extend([ - (os.chdir, 'cd %%s', self.tmpdir), - ]) - - for archive in archive_list: - if not os.path.isabs(archive): - archive = os.path.join(self.orig_cwd, archive) - if os.path.isdir(archive): - dest = os.path.split(archive)[1] - commands.append((shutil.copytree, 'cp -r %%s %%s', archive, dest)) - else: - suffix = self.archive_splitext(archive)[1] - unpack_command = self.unpack_map.get(suffix) - if not unpack_command: - dest = os.path.split(archive)[1] - commands.append((shutil.copyfile, 'cp %%s %%s', archive, dest)) - else: - commands.append(unpack_command + (archive,)) - - commands.extend([ - (os.chdir, 'cd %%s', self.subdir), - ]) - - commands.extend(self.initial_commands) - - commands.extend([ - (lambda: read_tree('.'), - 'find * -type f | xargs cat > /dev/null'), - - (self.set_env, 'export %%s=%%s', - 'SCONS_LIB_DIR', self.scons_lib_dir), - - '%(python)s %(scons_wrapper)s --version', - ]) - - index = 0 - for run_command in self.run_commands: - setattr(self, 'prof%d' % index, self.profile_name(index)) - c = ( - self.log_execute, - self.log_display, - run_command, - self.logfile_name(index), - ) - commands.append(c) - index = index + 1 - - commands.extend([ - (os.chdir, 'cd %%s', self.orig_cwd), - ]) - - if not os.environ.get('PRESERVE'): - commands.extend(removals) - commands.append((shutil.rmtree, 'rm -rf %%s', self.tmpdir)) - - self.run_command_list(commands, self.__dict__) - - self.scons = save_scons - self.scons_lib_dir = save_scons_lib_dir - self.scons_wrapper = save_scons_wrapper - - # - - def help_time(self): - help = """\ - Usage: scons-time time [OPTIONS] FILE [...] - - -C DIR, --chdir=DIR Change to DIR before looking for files - -f FILE, --file=FILE Read configuration from specified FILE - --fmt=FORMAT, --format=FORMAT Print data in specified FORMAT - -h, --help Print this help and exit - -p STRING, --prefix=STRING Use STRING as log file/profile prefix - -t NUMBER, --tail=NUMBER Only report the last NUMBER files - --which=TIMER Plot timings for TIMER: total, - SConscripts, SCons, commands. - """ - sys.stdout.write(self.outdent(help)) - sys.stdout.flush() - - def do_time(self, argv): - - format = 'ascii' - logfile_path = lambda x: x - tail = None - which = 'total' - - short_opts = '?C:f:hp:t:' - - long_opts = [ - 'chdir=', - 'file=', - 'fmt=', - 'format=', - 'help', - 'prefix=', - 'tail=', - 'title=', - 'which=', - ] - - opts, args = getopt.getopt(argv[1:], short_opts, long_opts) - - for o, a in opts: - if o in ('-C', '--chdir'): - self.chdir = a - elif o in ('-f', '--file'): - self.config_file = a - elif o in ('--fmt', '--format'): - format = a - elif o in ('-?', '-h', '--help'): - self.do_help(['help', 'time']) - sys.exit(0) - elif o in ('-p', '--prefix'): - self.prefix = a - elif o in ('-t', '--tail'): - tail = int(a) - elif o in ('--title',): - self.title = a - elif o in ('--which',): - if a not in list(self.time_strings.keys()): - sys.stderr.write('%s: time: Unrecognized timer "%s".\n' % (self.name, a)) - sys.stderr.write('%s Type "%s help time" for help.\n' % (self.name_spaces, self.name)) - sys.exit(1) - which = a - - if self.config_file: - with open(self.config_file, 'r') as f: - config = f.read() - HACK_for_exec(config, self.__dict__) - - if self.chdir: - os.chdir(self.chdir) - logfile_path = lambda x: os.path.join(self.chdir, x) - - if not args: - - pattern = '%s*.log' % self.prefix - args = self.args_to_files([pattern], tail) - - if not args: - if self.chdir: - directory = self.chdir - else: - directory = os.getcwd() - - sys.stderr.write('%s: time: No arguments specified.\n' % self.name) - sys.stderr.write('%s No %s*.log files found in "%s".\n' % (self.name_spaces, self.prefix, directory)) - sys.stderr.write('%s Type "%s help time" for help.\n' % (self.name_spaces, self.name)) - sys.exit(1) - - else: - - args = self.args_to_files(args, tail) - - cwd_ = os.getcwd() + os.sep - - if format == 'ascii': - - columns = ("Total", "SConscripts", "SCons", "commands") - self.ascii_table(args, columns, self.get_debug_times, logfile_path) - - elif format == 'gnuplot': - - results = self.collect_results(args, self.get_debug_times, - self.time_strings[which]) - - self.gnuplot_results(results, fmt='%s %.6f') - - else: - - sys.stderr.write('%s: time: Unknown format "%s".\n' % (self.name, format)) - sys.exit(1) - -if __name__ == '__main__': - opts, args = getopt.getopt(sys.argv[1:], 'h?V', ['help', 'version']) - - ST = SConsTimer() - - for o, a in opts: - if o in ('-?', '-h', '--help'): - ST.do_help(['help']) - sys.exit(0) - elif o in ('-V', '--version'): - sys.stdout.write('scons-time version\n') - sys.exit(0) - - if not args: - sys.stderr.write('Type "%s help" for usage.\n' % ST.name) - sys.exit(1) - - ST.execute_subcommand(args) - -# Local Variables: -# tab-width:4 -# indent-tabs-mode:nil -# End: -# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/testing/framework/TestSCons_time.py b/testing/framework/TestSCons_time.py index c6373eb..0573404 100644 --- a/testing/framework/TestSCons_time.py +++ b/testing/framework/TestSCons_time.py @@ -158,7 +158,7 @@ class TestSCons_time(TestCommon): self.orig_cwd = os.getcwd() try: - script_dir = os.environ['SCONS_SCRIPT_DIR'] + script_dir = os.environ['SCONS_TOOLS_DIR'] except KeyError: pass else: @@ -173,8 +173,6 @@ class TestSCons_time(TestCommon): if 'interpreter' not in kw: kw['interpreter'] = [python,] - if sys.version_info[0] < 3: - kw['interpreter'].append('-tt') if 'match' not in kw: kw['match'] = match_exact -- cgit v0.12 From 7b5438367cd25d1cb093a323d4b9c997cd26fd79 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 21 Mar 2020 21:41:14 -0700 Subject: Fix quoting on IMPLICIT_COMMAND_DEPENDENCIES=all --- test/Repository/StaticLibrary.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Repository/StaticLibrary.py b/test/Repository/StaticLibrary.py index 29cb841..c4a901b 100644 --- a/test/Repository/StaticLibrary.py +++ b/test/Repository/StaticLibrary.py @@ -30,7 +30,7 @@ import TestSCons _obj = TestSCons._obj _exe = TestSCons._exe -for implicit_deps in ['1', '2']: +for implicit_deps in ['0', '1', '2', '\"all\"']: test = TestSCons.TestSCons() # -- cgit v0.12 From 8a4074083f20090e9092f67b888275d4c83763a2 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sun, 22 Mar 2020 13:15:09 -0700 Subject: Change location of scons.py to scripts --- bootstrap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap.py b/bootstrap.py index d47c966..7d8528a 100755 --- a/bootstrap.py +++ b/bootstrap.py @@ -183,7 +183,7 @@ def main(): else: pass_through_args.append(arg) - scons_py = os.path.join('src', 'script', 'scons.py') + scons_py = os.path.join('scripts', 'scons.py') src_engine = os.path.join('src', 'engine') MANIFEST_in = find(os.path.join(src_engine, 'MANIFEST.in')) manifest_files = [os.path.join(src_engine, x) -- cgit v0.12 From 26b4d168832b27d89962c5faf9a2ed75b174abc8 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sun, 22 Mar 2020 13:15:39 -0700 Subject: Disable generating API docs for now epydoc is unmaintained and (?) not working with py3. Need to switch to sphinx(?) --- doc/SConscript | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/SConscript b/doc/SConscript index 8b22b3b..6a522d0 100644 --- a/doc/SConscript +++ b/doc/SConscript @@ -499,7 +499,7 @@ else: Local(css_file) if not skip_doc: - if not epydoc_cli and not epydoc: + if True: #not epydoc_cli and not epydoc: print("doc: epydoc not found, skipping building API documentation.") else: # XXX Should be in common with reading the same thing in -- cgit v0.12 From e836436961ed13cf430452bfe81016aea243dbe3 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sun, 22 Mar 2020 13:23:06 -0700 Subject: remove gentoo ebuild logic. distro packager has their own --- SConstruct | 10 +--------- gentoo/scons.ebuild.in | 26 -------------------------- 2 files changed, 1 insertion(+), 35 deletions(-) delete mode 100644 gentoo/scons.ebuild.in diff --git a/SConstruct b/SConstruct index f696499..97fda1d 100644 --- a/SConstruct +++ b/SConstruct @@ -486,15 +486,7 @@ Version_values = [Value(command_line.version), Value(command_line.build_id)] # '$PYTHON $PYTHONFLAGS "%s" install "--prefix=$TEST_TAR_GZ_DIR" --standalone-lib' % \ # os.path.join(unpack_tar_gz_dir, pkg_version, 'setup.py'), # ]) -# -# # -# # Generate portage files for submission to Gentoo Linux. -# # -# gentoo = os.path.join(build, 'gentoo') -# ebuild = os.path.join(gentoo, 'scons-%s.ebuild' % command_line.version) -# digest = os.path.join(gentoo, 'files', 'digest-scons-%s' % command_line.version) -# env.Command(ebuild, os.path.join('gentoo', 'scons.ebuild.in'), SCons_revision) -# +## # # def Digestify(target, source, env): # import hashlib diff --git a/gentoo/scons.ebuild.in b/gentoo/scons.ebuild.in deleted file mode 100644 index 50c9ad9..0000000 --- a/gentoo/scons.ebuild.in +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 1999-2003 Gentoo Technologies, Inc. -# Distributed under the terms of the GNU General Public License v2 -# $__NULL__Header: /home/cvsroot/gentoo-x86/dev-util/scons/scons-__VERSION__.ebuild,v 1.2 2003/02/13 12:00:11 vapier Exp __NULL__$ - -MY_P=${PN}-__VERSION__ -S=${WORKDIR}/${MY_P} -DESCRIPTION="Extensible python-based build utility" -SRC_URI="mirror://sourceforge/${PN}/${MY_P}.tar.gz" -HOMEPAGE="http://www.scons.org" - -SLOT="0" -LICENSE="as-is" -KEYWORDS="~x86 ~sparc" - -DEPEND=">=dev-lang/python-2.4" - -src_compile() { - python setup.py build -} - -src_install () { - python setup.py install --root=${D} - dodoc *.txt PKG-INFO MANIFEST - doman scons.1 - doman sconsign.1 -} -- cgit v0.12 From 559a661338b1c6988107de4a3caf8c49d2ee708d Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sun, 22 Mar 2020 21:01:46 -0700 Subject: add logic to not include *Tests.py in wheel --- MANIFEST.in | 6 +- SConstruct | 640 ------------------------------------------------- setup.py | 25 +- src/MANIFEST.in | 0 src/engine/MANIFEST.in | 186 -------------- src/setup.cfg | 8 - src/setup.py | 522 ---------------------------------------- 7 files changed, 26 insertions(+), 1361 deletions(-) delete mode 100644 src/MANIFEST.in delete mode 100644 src/engine/MANIFEST.in delete mode 100644 src/setup.cfg delete mode 100755 src/setup.py diff --git a/MANIFEST.in b/MANIFEST.in index 4e4ae5a..04aee14 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,5 @@ -recursive-exclude src/engine *Tests.py -global-exclude *Tests.py +global-exclude **Tests.py + recursive-include src/engine/SCons/Tool/docbook * + + diff --git a/SConstruct b/SConstruct index 97fda1d..a76471f 100644 --- a/SConstruct +++ b/SConstruct @@ -184,470 +184,6 @@ env = Environment( Version_values = [Value(command_line.version), Value(command_line.build_id)] -# -# Define SCons packages. -# -# In the original, more complicated packaging scheme, we were going -# to have separate packages for: -# -# python-scons only the build engine -# scons-script only the script -# scons the script plus the build engine -# -# We're now only delivering a single "scons" package, but this is still -# "built" as two sub-packages (the build engine and the script), so -# the definitions remain here, even though we're not using them for -# separate packages. -# - -# from distutils.sysconfig import get_python_lib -# -# python_scons = { -# 'pkg': 'python-' + project, -# 'src_subdir': 'engine', -# 'inst_subdir': get_python_lib(), -# -# 'files': ['LICENSE.txt', -# 'README.txt', -# 'setup.cfg', -# 'setup.py', -# ], -# -# 'filemap': { -# 'LICENSE.txt': '../LICENSE.txt' -# }, -# -# 'buildermap': {}, -# -# 'explicit_deps': { -# 'SCons/__init__.py': Version_values, -# }, -# } - -# scons_script = { -# 'pkg': project + '-script', -# 'src_subdir': 'script', -# 'inst_subdir': 'bin', -# -# 'files': [ -# 'LICENSE.txt', -# 'README.txt', -# 'setup.cfg', -# 'setup.py', -# ], -# -# 'filemap': { -# 'LICENSE.txt': '../LICENSE.txt', -# 'scons': 'scons.py', -# 'sconsign': 'sconsign.py', -# 'scons-time': 'scons-time.py', -# 'scons-configure-cache': 'scons-configure-cache.py', -# }, -# -# 'buildermap': {}, -# -# 'explicit_deps': { -# 'scons': Version_values, -# 'sconsign': Version_values, -# }, -# } - -# scons = { -# 'pkg': project, -# -# 'files': [ -# 'CHANGES.txt', -# 'LICENSE.txt', -# 'README.txt', -# 'RELEASE.txt', -# 'scons.1', -# 'sconsign.1', -# 'scons-time.1', -# # 'script/scons.bat', -# 'setup.cfg', -# 'setup.py', -# ], -# -# 'filemap': { -# 'scons.1': '$BUILDDIR/doc/man/scons.1', -# 'sconsign.1': '$BUILDDIR/doc/man/sconsign.1', -# 'scons-time.1': '$BUILDDIR/doc/man/scons-time.1', -# }, -# -# 'buildermap': { -# 'scons.1': env.SOElim, -# 'sconsign.1': env.SOElim, -# 'scons-time.1': env.SOElim, -# }, -# -# 'subpkgs': [python_scons], -# -# 'subinst_dirs': { -# 'python-' + project: python_project_subinst_dir, -# project + '-script': project_script_subinst_dir, -# }, -# } -# -# scripts = ['scons', 'sconsign', 'scons-time', 'scons-configure-cache'] -# -# src_deps = [] -# src_files = [] -# -# for p in [scons]: -# # -# # Initialize variables with the right directories for this package. -# # -# pkg = p['pkg'] -# pkg_version = "%s-%s" % (pkg, command_line.version) -# -# src = 'src' -# if 'src_subdir' in p: -# src = os.path.join(src, p['src_subdir']) -# -# build = os.path.join(command_line.build_dir, pkg) -# -# tar_gz = os.path.join(build, 'dist', "%s.tar.gz" % pkg_version) -# platform_tar_gz = os.path.join(build, -# 'dist', -# "%s.%s.tar.gz" % (pkg_version, platform)) -# zip = os.path.join(build, 'dist', "%s.zip" % pkg_version) -# platform_zip = os.path.join(build, -# 'dist', -# "%s.%s.zip" % (pkg_version, platform)) -# -# # -# # Update the environment with the relevant information -# # for this package. -# # -# # We can get away with calling setup.py using a directory path -# # like this because we put a preamble in it that will chdir() -# # to the directory in which setup.py exists. -# # -# setup_py = os.path.join(build, 'setup.py') -# env.Replace(PKG=pkg, -# PKG_VERSION=pkg_version, -# SETUP_PY='"%s"' % setup_py) -# Local(setup_py) -# -# # -# # Read up the list of source files from our MANIFEST.in. -# # This list should *not* include LICENSE.txt, MANIFEST, -# # README.txt, or setup.py. Make a copy of the list for the -# # destination files. -# # -# manifest_in = File(os.path.join(src, 'MANIFEST.in')).rstr() -# src_files = bootstrap.parseManifestLines(src, manifest_in) -# raw_files = src_files[:] -# dst_files = src_files[:] -# -# MANIFEST_in_list = [] -# -# if 'subpkgs' in p: -# # -# # This package includes some sub-packages. Read up their -# # MANIFEST.in files, and add them to our source and destination -# # file lists, modifying them as appropriate to add the -# # specified subdirs. -# # -# for sp in p['subpkgs']: -# ssubdir = sp['src_subdir'] -# isubdir = p['subinst_dirs'][sp['pkg']] -# -# MANIFEST_in = File(os.path.join(src, ssubdir, 'MANIFEST.in')).rstr() -# MANIFEST_in_list.append(MANIFEST_in) -# files = bootstrap.parseManifestLines(os.path.join(src, ssubdir), MANIFEST_in) -# -# raw_files.extend(files) -# src_files.extend([os.path.join(ssubdir, x) for x in files]) -# -# files = [os.path.join(isubdir, x) for x in files] -# dst_files.extend(files) -# for k, f in sp['filemap'].items(): -# if f: -# k = os.path.join(ssubdir, k) -# p['filemap'][k] = os.path.join(ssubdir, f) -# for f, deps in sp['explicit_deps'].items(): -# f = os.path.join(build, ssubdir, f) -# env.Depends(f, deps) -# -# # -# # Now that we have the "normal" source files, add those files -# # that are standard for each distribution. Note that we don't -# # add these to dst_files, because they don't get installed. -# # And we still have the MANIFEST to add. -# # -# src_files.extend(p['files']) -# -# # -# # Now run everything in src_file through the sed command we -# # concocted to expand __FILE__, __VERSION__, etc. -# # -# for b in src_files: -# s = p['filemap'].get(b, b) -# if not s[0] == '$' and not os.path.isabs(s): -# s = os.path.join(src, s) -# -# builder = p['buildermap'].get(b, env.SCons_revision) -# x = builder(os.path.join(build, b), s) -# -# Local(x) -# -# # -# # NOW, finally, we can create the MANIFEST, which we do -# # by having Python spit out the contents of the src_files -# # array we've carefully created. After we've added -# # MANIFEST itself to the array, of course. -# # -# src_files.append("MANIFEST") -# MANIFEST_in_list.append(os.path.join(src, 'MANIFEST.in')) -# -# -# def write_src_files(target, source, **kw): -# global src_files -# src_files.sort() -# with open(str(target[0]), 'w') as f: -# for file in src_files: -# f.write(file + "\n") -# return 0 -# -# -# env.Command(os.path.join(build, 'MANIFEST'), -# MANIFEST_in_list, -# write_src_files) -# -# # -# # Now go through and arrange to create whatever packages we can. -# # -# build_src_files = [os.path.join(build, x) for x in src_files] -# Local(*build_src_files) -# -# distutils_formats = [] -# distutils_targets = [] -# dist_distutils_targets = [] -# -# for target in distutils_targets: -# dist_target = env.Install('$DISTDIR', target) -# AddPostAction(dist_target, Chmod(dist_target, 0o644)) -# dist_distutils_targets += dist_target -# -# if not gzip: -# print("gzip not found in %s; skipping .tar.gz package for %s." % (os.environ['PATH'], pkg)) -# else: -# -# distutils_formats.append('gztar') -# -# src_deps.append(tar_gz) -# -# distutils_targets.extend([tar_gz, platform_tar_gz]) -# -# dist_tar_gz = env.Install('$DISTDIR', tar_gz) -# dist_platform_tar_gz = env.Install('$DISTDIR', platform_tar_gz) -# Local(dist_tar_gz, dist_platform_tar_gz) -# AddPostAction(dist_tar_gz, Chmod(dist_tar_gz, 0o644)) -# AddPostAction(dist_platform_tar_gz, Chmod(dist_platform_tar_gz, 0o644)) -# -# # -# # Unpack the tar.gz archive created by the distutils into -# # build/unpack-tar-gz/scons-{version}. -# # -# # We'd like to replace the last three lines with the following: -# # -# # tar zxf $SOURCES -C $UNPACK_TAR_GZ_DIR -# # -# # but that gives heartburn to Cygwin's tar, so work around it -# # with separate zcat-tar-rm commands. -# # -# unpack_tar_gz_files = [os.path.join(unpack_tar_gz_dir, pkg_version, x) -# for x in src_files] -# env.Command(unpack_tar_gz_files, dist_tar_gz, [ -# Delete(os.path.join(unpack_tar_gz_dir, pkg_version)), -# "$ZCAT $SOURCES > .temp", -# "tar xf .temp -C $UNPACK_TAR_GZ_DIR", -# Delete(".temp"), -# ]) -# -# # -# # Run setup.py in the unpacked subdirectory to "install" everything -# # into our build/test subdirectory. The runtest.py script will set -# # PYTHONPATH so that the tests only look under build/test-{package}, -# # and under testing/framework (for the testing modules TestCmd.py, TestSCons.py, -# # etc.). This makes sure that our tests pass with what -# # we really packaged, not because of something hanging around in -# # the development directory. -# # -# # We can get away with calling setup.py using a directory path -# # like this because we put a preamble in it that will chdir() -# # to the directory in which setup.py exists. -# # -# dfiles = [os.path.join(test_tar_gz_dir, x) for x in dst_files] -# env.Command(dfiles, unpack_tar_gz_files, [ -# Delete(os.path.join(unpack_tar_gz_dir, pkg_version, 'build')), -# Delete("$TEST_TAR_GZ_DIR"), -# '$PYTHON $PYTHONFLAGS "%s" install "--prefix=$TEST_TAR_GZ_DIR" --standalone-lib' % \ -# os.path.join(unpack_tar_gz_dir, pkg_version, 'setup.py'), -# ]) -## -# -# def Digestify(target, source, env): -# import hashlib -# src = source[0].rfile() -# with open(str(src), 'rb') as f: -# contents = f.read() -# m = hashlib.md5() -# m.update(contents) -# sig = m.hexdigest() -# bytes = os.stat(str(src))[6] -# with open(str(target[0]), 'w') as f: -# f.write("MD5 %s %s %d\n" % (sig, src.name, bytes)) -# -# -# env.Command(digest, tar_gz, Digestify) -# -# if not zipit: -# print("zip not found; skipping .zip package for %s." % pkg) -# else: -# -# distutils_formats.append('zip') -# -# src_deps.append(zip) -# -# distutils_targets.extend([zip, platform_zip]) -# -# dist_zip = env.Install('$DISTDIR', zip) -# dist_platform_zip = env.Install('$DISTDIR', platform_zip) -# Local(dist_zip, dist_platform_zip) -# AddPostAction(dist_zip, Chmod(dist_zip, 0o644)) -# AddPostAction(dist_platform_zip, Chmod(dist_platform_zip, 0o644)) -# -# # -# # Unpack the zip archive created by the distutils into -# # build/unpack-zip/scons-{version}. -# # -# unpack_zip_files = [os.path.join(unpack_zip_dir, pkg_version, x) -# for x in src_files] -# -# env.Command(unpack_zip_files, dist_zip, [ -# Delete(os.path.join(unpack_zip_dir, pkg_version)), -# unzipit, -# ]) -# -# # -# # Run setup.py in the unpacked subdirectory to "install" everything -# # into our build/test subdirectory. The runtest.py script will set -# # PYTHONPATH so that the tests only look under build/test-{package}, -# # and under testing/framework (for the testing modules TestCmd.py, TestSCons.py, -# # etc.). This makes sure that our tests pass with what -# # we really packaged, not because of something hanging around in -# # the development directory. -# # -# # We can get away with calling setup.py using a directory path -# # like this because we put a preamble in it that will chdir() -# # to the directory in which setup.py exists. -# # -# dfiles = [os.path.join(test_zip_dir, x) for x in dst_files] -# env.Command(dfiles, unpack_zip_files, [ -# Delete(os.path.join(unpack_zip_dir, pkg_version, 'build')), -# Delete("$TEST_ZIP_DIR"), -# '$PYTHON $PYTHONFLAGS "%s" install "--prefix=$TEST_ZIP_DIR" --standalone-lib' % \ -# os.path.join(unpack_zip_dir, pkg_version, 'setup.py'), -# ]) -# -# # -# # Use the Python distutils to generate the appropriate packages. -# # -# commands = [ -# Delete(os.path.join(build, 'build', 'lib')), -# Delete(os.path.join(build, 'build', 'scripts')), -# ] -# -# if distutils_formats: -# commands.append(Delete(os.path.join(build, -# 'build', -# 'bdist.' + platform, -# 'dumb'))) -# for format in distutils_formats: -# commands.append("$PYTHON $PYTHONFLAGS $SETUP_PY bdist_dumb -f %s" % format) -# -# commands.append("$PYTHON $PYTHONFLAGS $SETUP_PY sdist --formats=%s" % \ -# ','.join(distutils_formats)) -# -# env.Command(distutils_targets, build_src_files, commands) -# -# # -# # Now create local packages for people who want to let people -# # build their SCons-buildable packages without having to -# # install SCons. -# # -# s_l_v = '%s-local-%s' % (pkg, command_line.version) -# -# local = pkg + '-local' -# build_dir_local = os.path.join(command_line.build_dir, local) -# build_dir_local_slv = os.path.join(command_line.build_dir, local, s_l_v) -# -# dist_local_tar_gz = os.path.join("$DISTDIR/%s.tar.gz" % s_l_v) -# dist_local_zip = os.path.join("$DISTDIR/%s.zip" % s_l_v) -# AddPostAction(dist_local_tar_gz, Chmod(dist_local_tar_gz, 0o644)) -# AddPostAction(dist_local_zip, Chmod(dist_local_zip, 0o644)) -# -# commands = [ -# Delete(build_dir_local), -# '$PYTHON $PYTHONFLAGS $SETUP_PY install "--install-script=%s" "--install-lib=%s" --no-install-man --no-compile --standalone-lib --no-version-script' % \ -# (build_dir_local, build_dir_local_slv), -# ] -# -# for script in scripts: -# # add .py extension for scons-local scripts on non-windows platforms -# if is_windows(): -# break -# local_script = os.path.join(build_dir_local, script) -# commands.append(Move(local_script + '.py', local_script)) -# -# rf = [x for x in raw_files if not x in scripts] -# rf = [os.path.join(s_l_v, x) for x in rf] -# for script in scripts: -# rf.append("%s.py" % script) -# local_targets = [os.path.join(build_dir_local, x) for x in rf] -# -# env.Command(local_targets, build_src_files, commands) -# -# scons_LICENSE = os.path.join(build_dir_local, 'scons-LICENSE') -# l = env.SCons_revision(scons_LICENSE, 'LICENSE-local') -# local_targets.append(l) -# Local(l) -# -# scons_README = os.path.join(build_dir_local, 'scons-README') -# l = env.SCons_revision(scons_README, 'README-local') -# local_targets.append(l) -# Local(l) -# -# if gzip: -# if is_windows(): -# # avoid problem with tar interpreting c:/ as a remote machine -# tar_cargs = '-cz --force-local -f' -# else: -# tar_cargs = '-czf' -# env.Command(dist_local_tar_gz, -# local_targets, -# "cd %s && tar %s $( ${TARGET.abspath} $) *" % (build_dir_local, tar_cargs)) -# -# unpack_targets = [os.path.join(test_local_tar_gz_dir, x) for x in rf] -# commands = [Delete(test_local_tar_gz_dir), -# Mkdir(test_local_tar_gz_dir), -# "cd %s && tar xzf $( ${SOURCE.abspath} $)" % test_local_tar_gz_dir] -# -# env.Command(unpack_targets, dist_local_tar_gz, commands) -# -# if zipit: -# env.Command(dist_local_zip, local_targets, zipit, -# CD=build_dir_local, PSV='.') -# -# unpack_targets = [os.path.join(test_local_zip_dir, x) for x in rf] -# commands = [Delete(test_local_zip_dir), -# Mkdir(test_local_zip_dir), -# unzipit] -# -# env.Command(unpack_targets, dist_local_zip, unzipit, -# UNPACK_ZIP_DIR=test_local_zip_dir) # # @@ -659,179 +195,3 @@ Export('command_line', 'env', 'whereis', 'revaction') SConscript('doc/SConscript') -# # -# # If we're running in a Git working directory, pack up a complete -# # source archive from the project files and files in the change. -# # -# -# -# sfiles = [l.split()[-1] for l in command_line.git_status_lines] -# if command_line.git_status_lines: -# # slines = [l for l in git_status_lines if 'modified:' in l] -# # sfiles = [l.split()[-1] for l in slines] -# pass -# else: -# print("Not building in a Git tree; skipping building src package.") -# -# if sfiles: -# remove_patterns = [ -# '*.gitignore', -# '*.hgignore', -# 'www/*', -# ] -# -# for p in remove_patterns: -# sfiles = [s for s in sfiles if not fnmatch.fnmatch(s, p)] -# -# if sfiles: -# ps = "%s-src" % project -# psv = "%s-%s" % (ps, command_line.version) -# b_ps = os.path.join(command_line.build_dir, ps) -# b_psv = os.path.join(command_line.build_dir, psv) -# b_psv_stamp = b_psv + '-stamp' -# -# src_tar_gz = os.path.join(command_line.build_dir, 'dist', '%s.tar.gz' % psv) -# src_zip = os.path.join(command_line.build_dir, 'dist', '%s.zip' % psv) -# -# Local(src_tar_gz, src_zip) -# -# for file in sfiles: -# if file.endswith('jpg') or file.endswith('png'): -# # don't revision binary files. -# env.Install(os.path.dirname(os.path.join(b_ps, file)), file) -# else: -# env.SCons_revision(os.path.join(b_ps, file), file) -# -# b_ps_files = [os.path.join(b_ps, x) for x in sfiles] -# cmds = [ -# Delete(b_psv), -# Copy(b_psv, b_ps), -# Touch("$TARGET"), -# ] -# -# env.Command(b_psv_stamp, src_deps + b_ps_files, cmds) -# -# Local(*b_ps_files) -# -# if gzip: -# env.Command(src_tar_gz, b_psv_stamp, -# "tar cz${TAR_HFLAG} -f $TARGET -C build %s" % psv) -# -# # -# # Unpack the archive into build/unpack/scons-{version}. -# # -# unpack_tar_gz_files = [os.path.join(unpack_tar_gz_dir, psv, x) -# for x in sfiles] -# -# # -# # We'd like to replace the last three lines with the following: -# # -# # tar zxf $SOURCES -C $UNPACK_TAR_GZ_DIR -# # -# # but that gives heartburn to Cygwin's tar, so work around it -# # with separate zcat-tar-rm commands. -# env.Command(unpack_tar_gz_files, src_tar_gz, [ -# Delete(os.path.join(unpack_tar_gz_dir, psv)), -# "$ZCAT $SOURCES > .temp", -# "tar xf .temp -C $UNPACK_TAR_GZ_DIR", -# Delete(".temp"), -# ]) -# -# # -# # Run setup.py in the unpacked subdirectory to "install" everything -# # into our build/test subdirectory. The runtest.py script will set -# # PYTHONPATH so that the tests only look under build/test-{package}, -# # and under testing/framework (for the testing modules TestCmd.py, -# # TestSCons.py, etc.). This makes sure that our tests pass with -# # what we really packaged, not because of something hanging around -# # in the development directory. -# # -# # We can get away with calling setup.py using a directory path -# # like this because we put a preamble in it that will chdir() -# # to the directory in which setup.py exists. -# # -# dfiles = [os.path.join(test_src_tar_gz_dir, x) for x in dst_files] -# scons_lib_dir = os.path.join(unpack_tar_gz_dir, psv, 'src', 'engine') -# ENV = env.Dictionary('ENV').copy() -# ENV['SCONS_LIB_DIR'] = scons_lib_dir -# ENV['USERNAME'] = command_line.developer -# env.Command(dfiles, unpack_tar_gz_files, -# [ -# Delete(os.path.join(unpack_tar_gz_dir, -# psv, -# 'build', -# 'scons', -# 'build')), -# Delete("$TEST_SRC_TAR_GZ_DIR"), -# 'cd "%s" && $PYTHON $PYTHONFLAGS "%s" "%s" VERSION="$VERSION"' % \ -# (os.path.join(unpack_tar_gz_dir, psv), -# os.path.join('src', 'script', 'scons.py'), -# os.path.join('build', 'scons')), -# '$PYTHON $PYTHONFLAGS "%s" install "--prefix=$TEST_SRC_TAR_GZ_DIR" --standalone-lib' % \ -# os.path.join(unpack_tar_gz_dir, -# psv, -# 'build', -# 'scons', -# 'setup.py'), -# ], -# ENV=ENV) -# -# if zipit: -# env.Command(src_zip, b_psv_stamp, zipit, CD='build', PSV=psv) -# -# # -# # Unpack the archive into build/unpack/scons-{version}. -# # -# unpack_zip_files = [os.path.join(unpack_zip_dir, psv, x) -# for x in sfiles] -# -# env.Command(unpack_zip_files, src_zip, [ -# Delete(os.path.join(unpack_zip_dir, psv)), -# unzipit -# ]) -# -# # -# # Run setup.py in the unpacked subdirectory to "install" everything -# # into our build/test subdirectory. The runtest.py script will set -# # PYTHONPATH so that the tests only look under build/test-{package}, -# # and under testing/framework (for the testing modules TestCmd.py, -# # TestSCons.py, etc.). This makes sure that our tests pass with -# # what we really packaged, not because of something hanging -# # around in the development directory. -# # -# # We can get away with calling setup.py using a directory path -# # like this because we put a preamble in it that will chdir() -# # to the directory in which setup.py exists. -# # -# dfiles = [os.path.join(test_src_zip_dir, x) for x in dst_files] -# scons_lib_dir = os.path.join(unpack_zip_dir, psv, 'src', 'engine') -# ENV = env.Dictionary('ENV').copy() -# ENV['SCONS_LIB_DIR'] = scons_lib_dir -# ENV['USERNAME'] = command_line.developer -# env.Command(dfiles, unpack_zip_files, -# [ -# Delete(os.path.join(unpack_zip_dir, -# psv, -# 'build', -# 'scons', -# 'build')), -# Delete("$TEST_SRC_ZIP_DIR"), -# 'cd "%s" && $PYTHON $PYTHONFLAGS "%s" "%s" VERSION="$VERSION"' % \ -# (os.path.join(unpack_zip_dir, psv), -# os.path.join('src', 'script', 'scons.py'), -# os.path.join('build', 'scons')), -# '$PYTHON $PYTHONFLAGS "%s" install "--prefix=$TEST_SRC_ZIP_DIR" --standalone-lib' % \ -# os.path.join(unpack_zip_dir, -# psv, -# 'build', -# 'scons', -# 'setup.py'), -# ], -# ENV=ENV) -# -# for pf, help_text in packaging_flavors: -# Alias(pf, [ -# os.path.join(command_line.build_dir, 'test-' + pf), -# os.path.join(command_line.build_dir, 'testing/framework'), -# os.path.join(command_line.build_dir, 'runtest.py'), -# ]) diff --git a/setup.py b/setup.py index ff975d4..349a24e 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,25 @@ -from setuptools import setup,find_packages +import fnmatch +from setuptools import find_packages, setup +from setuptools.command.build_py import build_py as build_py_orig + + +exclude = ['*Tests'] + + +class build_py(build_py_orig): + + def find_package_modules(self, package, package_dir): + """ + Custom module to find package modules. + It will strip out any modules which match the glob patters in exclude above + """ + modules = super().find_package_modules(package, package_dir) + return [(pkg, mod, file, ) for (pkg, mod, file, ) in modules + if not any(fnmatch.fnmatchcase(mod, pat=pattern) + for pattern in exclude)] setup( - # packages = find_packages("src/engine"), - # package_dir={"":"src/engine"}, + cmdclass={ + 'build_py': build_py, + } ) \ No newline at end of file diff --git a/src/MANIFEST.in b/src/MANIFEST.in deleted file mode 100644 index e69de29..0000000 diff --git a/src/engine/MANIFEST.in b/src/engine/MANIFEST.in deleted file mode 100644 index 3c2c0c2..0000000 --- a/src/engine/MANIFEST.in +++ /dev/null @@ -1,186 +0,0 @@ -SCons/__init__.py -SCons/__main__.py -SCons/Action.py -SCons/Builder.py -SCons/compat/*.py -SCons/CacheDir.py -SCons/Conftest.py -SCons/cpp.py -SCons/dblite.py -SCons/Debug.py -SCons/Defaults.py -SCons/Environment.py -SCons/Errors.py -SCons/Executor.py -SCons/Job.py -SCons/exitfuncs.py -SCons/Memoize.py -SCons/Node/__init__.py -SCons/Node/Alias.py -SCons/Node/FS.py -SCons/Node/Python.py -SCons/PathList.py -SCons/Platform/__init__.py -SCons/Platform/aix.py -SCons/Platform/cygwin.py -SCons/Platform/darwin.py -SCons/Platform/hpux.py -SCons/Platform/irix.py -SCons/Platform/os2.py -SCons/Platform/mingw.py -SCons/Platform/posix.py -SCons/Platform/sunos.py -SCons/Platform/win32.py -SCons/Platform/virtualenv.py -SCons/Scanner/__init__.py -SCons/Scanner/C.py -SCons/Scanner/D.py -SCons/Scanner/Dir.py -SCons/Scanner/Fortran.py -SCons/Scanner/IDL.py -SCons/Scanner/LaTeX.py -SCons/Scanner/Prog.py -SCons/Scanner/Python.py -SCons/Scanner/RC.py -SCons/Scanner/SWIG.py -SCons/SConf.py -SCons/SConsign.py -SCons/Script/__init__.py -SCons/Script/Interactive.py -SCons/Script/Main.py -SCons/Script/SConscript.py -SCons/Script/SConsOptions.py -SCons/Subst.py -SCons/Taskmaster.py -SCons/Tool/__init__.py -SCons/Tool/386asm.py -SCons/Tool/aixc++.py -SCons/Tool/aixcxx.py -SCons/Tool/aixcc.py -SCons/Tool/aixf77.py -SCons/Tool/aixlink.py -SCons/Tool/applelink.py -SCons/Tool/ar.py -SCons/Tool/as.py -SCons/Tool/bcc32.py -SCons/Tool/c++.py -SCons/Tool/cxx.py -SCons/Tool/cc.py -SCons/Tool/cyglink.py -SCons/Tool/clangCommon/__init__.py -SCons/Tool/clang.py -SCons/Tool/clangxx.py -SCons/Tool/cvf.py -SCons/Tool/DCommon.py -SCons/Tool/default.py -SCons/Tool/dmd.py -SCons/Tool/docbook/__init__.py -SCons/Tool/dvi.py -SCons/Tool/dvipdf.py -SCons/Tool/dvips.py -SCons/Tool/f03.py -SCons/Tool/f08.py -SCons/Tool/f77.py -SCons/Tool/f90.py -SCons/Tool/f95.py -SCons/Tool/filesystem.py -SCons/Tool/fortran.py -SCons/Tool/FortranCommon.py -SCons/Tool/g++.py -SCons/Tool/gxx.py -SCons/Tool/g77.py -SCons/Tool/gas.py -SCons/Tool/gcc.py -SCons/Tool/gdc.py -SCons/Tool/gfortran.py -SCons/Tool/gnulink.py -SCons/Tool/gs.py -SCons/Tool/hpc++.py -SCons/Tool/hpcxx.py -SCons/Tool/hpcc.py -SCons/Tool/hplink.py -SCons/Tool/icc.py -SCons/Tool/icl.py -SCons/Tool/ifl.py -SCons/Tool/ifort.py -SCons/Tool/ilink.py -SCons/Tool/ilink32.py -SCons/Tool/install.py -SCons/Tool/intelc.py -SCons/Tool/ipkg.py -SCons/Tool/jar.py -SCons/Tool/JavaCommon.py -SCons/Tool/javac.py -SCons/Tool/javah.py -SCons/Tool/latex.py -SCons/Tool/ldc.py -SCons/Tool/lex.py -SCons/Tool/link.py -SCons/Tool/linkloc.py -SCons/Tool/MSCommon/__init__.py -SCons/Tool/MSCommon/arch.py -SCons/Tool/MSCommon/common.py -SCons/Tool/MSCommon/netframework.py -SCons/Tool/MSCommon/sdk.py -SCons/Tool/MSCommon/vs.py -SCons/Tool/MSCommon/vc.py -SCons/Tool/m4.py -SCons/Tool/masm.py -SCons/Tool/midl.py -SCons/Tool/mingw.py -SCons/Tool/mslib.py -SCons/Tool/mslink.py -SCons/Tool/mssdk.py -SCons/Tool/msvc.py -SCons/Tool/msvs.py -SCons/Tool/mwcc.py -SCons/Tool/mwld.py -SCons/Tool/nasm.py -SCons/Tool/packaging/*.py -SCons/Tool/pdf.py -SCons/Tool/pdflatex.py -SCons/Tool/pdftex.py -SCons/Tool/PharLapCommon.py -SCons/Tool/python.py -SCons/Tool/qt.py -SCons/Tool/rmic.py -SCons/Tool/rpcgen.py -SCons/Tool/rpm.py -SCons/Tool/rpmutils.py -SCons/Tool/sgiar.py -SCons/Tool/sgic++.py -SCons/Tool/sgicxx.py -SCons/Tool/sgicc.py -SCons/Tool/sgilink.py -SCons/Tool/sunar.py -SCons/Tool/sunc++.py -SCons/Tool/suncxx.py -SCons/Tool/suncc.py -SCons/Tool/sunf77.py -SCons/Tool/sunf90.py -SCons/Tool/sunf95.py -SCons/Tool/sunlink.py -SCons/Tool/swig.py -SCons/Tool/tar.py -SCons/Tool/tex.py -SCons/Tool/textfile.py -SCons/Tool/tlib.py -SCons/Tool/wix.py -SCons/Tool/yacc.py -SCons/Tool/zip.py -SCons/Util.py -SCons/Variables/__init__.py -SCons/Variables/BoolVariable.py -SCons/Variables/EnumVariable.py -SCons/Variables/ListVariable.py -SCons/Variables/PackageVariable.py -SCons/Variables/PathVariable.py -SCons/Warnings.py -SCons/Tool/GettextCommon.py -SCons/Tool/gettext_tool.py -SCons/Tool/msgfmt.py -SCons/Tool/msginit.py -SCons/Tool/msgmerge.py -SCons/Tool/xgettext.py -#SCons/Tool/docbook/docbook-xsl-1.76.1/** -#SCons/Tool/docbook/utils/** diff --git a/src/setup.cfg b/src/setup.cfg deleted file mode 100644 index bda7571..0000000 --- a/src/setup.cfg +++ /dev/null @@ -1,8 +0,0 @@ -[bdist_rpm] -group = Development/Tools - -[bdist_wininst] -title = SCons - a software construction tool - -[bdist_wheel] -universal=1 diff --git a/src/setup.py b/src/setup.py deleted file mode 100755 index bea888b..0000000 --- a/src/setup.py +++ /dev/null @@ -1,522 +0,0 @@ -# -# __COPYRIGHT__ -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -import distutils.command.build_scripts -import distutils.command.install_scripts -import distutils.command.install_lib -import distutils.command.install_data -import distutils.command.install -import distutils.core -import distutils -import setuptools -""" -NOTE: Installed SCons is not importable like usual Python packages. It is - executed explicitly with command line scripts. This allows multiple - SCons versions to coexist within single Python installation, which - is critical for enterprise build cases. Explicit invokation is - necessary to avoid confusion over which version of SCons is active. - - By default SCons is installed into versioned directory, e.g. - site-packages/scons-2.1.0.alpha.20101125 and much of the stuff - below is dedicated to make it happen on various platforms. -""" - - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - -import os -import stat -import sys - -Version = "__VERSION__" - -man_pages = [ - 'scons.1', - 'sconsign.1', - 'scons-time.1', -] - -# change to setup.py directory if it was executed from other dir -(head, tail) = os.path.split(sys.argv[0]) -if head: - os.chdir(head) - sys.argv[0] = tail - -# flag if setup.py is run on win32 or _for_ win32 platform, -# (when building windows installer on linux, for example) -is_win32 = 0 -if not sys.platform == 'win32': - try: - if sys.argv[1] == 'bdist_wininst': - is_win32 = 1 - except IndexError: - pass -else: - is_win32 = 1 - - -_install = distutils.command.install.install -_install_data = distutils.command.install_data.install_data -_install_lib = distutils.command.install_lib.install_lib -_install_scripts = distutils.command.install_scripts.install_scripts -_build_scripts = distutils.command.build_scripts.build_scripts - - -class _options(object): - pass - - -Options = _options() - -Installed = [] - - -def set_explicitly(name, args): - """ - Return if the installation directory was set explicitly by the - user on the command line. This is complicated by the fact that - "install --install-lib=/foo" gets turned into "install_lib - --install-dir=/foo" internally. - """ - if args[0] == "install_" + name: - s = "--install-dir=" - else: - # The command is something else (usually "install") - s = "--install-%s=" % name - set = 0 - length = len(s) - for a in args[1:]: - if a[:length] == s: - set = 1 - break - return set - - -class install(_install): - user_options = _install.user_options + [ - ('no-scons-script', None, - "don't install 'scons', only install 'scons-%s'" % Version), - ('no-version-script', None, - "don't install 'scons-%s', only install 'scons'" % Version), - ('install-bat', None, - "install 'scons.bat' script"), - ('no-install-bat', None, - "do not install 'scons.bat' script"), - ('install-man', None, - "install SCons man pages"), - ('no-install-man', None, - "do not install SCons man pages"), - ('standard-lib', None, - "install SCons library in standard Python location"), - ('standalone-lib', None, - "install SCons library in separate standalone directory"), - ('version-lib', None, - "install SCons library in version-numbered directory"), - ] - boolean_options = _install.boolean_options + [ - 'no-scons-script', - 'no-version-script', - 'install-bat', - 'no-install-bat', - 'install-man', - 'no-install-man', - 'standard-lib', - 'standalone-lib', - 'version-lib' - ] - - if hasattr(os, 'link'): - user_options.append( - ('hardlink-scons', None, - "hard link 'scons' to the version-numbered script, don't make a separate 'scons' copy"), - ) - boolean_options.append('hardlink-script') - - if hasattr(os, 'symlink'): - user_options.append( - ('symlink-scons', None, - "make 'scons' a symbolic link to the version-numbered script, don't make a separate 'scons' copy"), - ) - boolean_options.append('symlink-script') - - def initialize_options(self): - _install.initialize_options(self) - self.no_scons_script = 0 - self.no_version_script = 0 - self.install_bat = 0 - self.no_install_bat = False # not is_win32 - self.install_man = 0 - self.standard_lib = 0 - self.standalone_lib = 0 - self.version_lib = 0 - self.hardlink_scons = 0 - self.symlink_scons = 0 - # Don't warn about having to put the library directory in the - # search path. - self.warn_dir = 0 - - def finalize_options(self): - _install.finalize_options(self) - if self.install_bat: - Options.install_bat = 1 - else: - Options.install_bat = not self.no_install_bat - if self.install_man: - Options.install_man = 1 - else: - Options.install_man = not self.no_install_man - Options.standard_lib = self.standard_lib - Options.standalone_lib = self.standalone_lib - Options.version_lib = self.version_lib - Options.install_scons_script = not self.no_scons_script - Options.install_version_script = not self.no_version_script - Options.hardlink_scons = self.hardlink_scons - Options.symlink_scons = self.symlink_scons - - -def get_scons_prefix(libdir, is_win32): - """ - Return the right prefix for SCons library installation. Find - this by starting with the library installation directory - (.../site-packages, most likely) and crawling back up until we reach - a directory name beginning with "python" (or "Python"). - """ - drive, head = os.path.splitdrive(libdir) - while head: - if head == os.sep: - break - head, tail = os.path.split(head) - if tail.lower()[:6] == "python": - # Found the Python library directory... - if is_win32: - # ...on Win32 systems, "scons" goes in the directory: - # C:\PythonXX => C:\PythonXX\scons - return os.path.join(drive + head, tail) - else: - # ...on other systems, "scons" goes above the directory: - # /usr/lib/pythonX.X => /usr/lib/scons - return os.path.join(drive + head) - return libdir - - -def force_to_usr_local(self): - """ - A hack to decide if we need to "force" the installation directories - to be under /usr/local. This is because Mac Os X Tiger and - Leopard, by default, put the libraries and scripts in their own - directories under /Library or /System/Library. - """ - return (sys.platform[:6] == 'darwin' and - (self.install_dir[:9] == '/Library/' or - self.install_dir[:16] == '/System/Library/')) - - -class install_lib(_install_lib): - def finalize_options(self): - _install_lib.finalize_options(self) - if force_to_usr_local(self): - self.install_dir = '/usr/local/lib' - args = self.distribution.script_args - if not set_explicitly("lib", args): - # They didn't explicitly specify the installation - # directory for libraries... - is_win32 = sys.platform == "win32" or args[0] == 'bdist_wininst' - prefix = get_scons_prefix(self.install_dir, is_win32) - if Options.standalone_lib: - # ...but they asked for a standalone directory. - self.install_dir = os.path.join(prefix, "scons") - elif Options.version_lib: - # ...they asked for a version-specific directory, - self.install_dir = os.path.join(prefix, "scons-%s" % Version) - elif not Options.standard_lib: - # default. - self.install_dir = os.path.join(prefix, "scons") - - msg = "Installed SCons library modules into %s" % self.install_dir - Installed.append(msg) - - -class install_scripts(_install_scripts): - def finalize_options(self): - _install_scripts.finalize_options(self) - if force_to_usr_local(self): - self.install_dir = '/usr/local/bin' - self.build_dir = os.path.join('build', 'scripts') - msg = "Installed SCons scripts into %s" % self.install_dir - Installed.append(msg) - - def do_nothing(self, *args, **kw): - pass - - def hardlink_scons(self, src, dst, ver): - try: - os.unlink(dst) - except OSError: - pass - os.link(ver, dst) - - def symlink_scons(self, src, dst, ver): - try: - os.unlink(dst) - except OSError: - pass - os.symlink(os.path.split(ver)[1], dst) - - def copy_scons(self, src, dst, *args): - try: - os.unlink(dst) - except OSError: - pass - self.copy_file(src, dst) - self.outfiles.append(dst) - - def run(self): - # --- distutils copy/paste --- - if not self.skip_build: - self.run_command('build_scripts') - # --- /distutils copy/paste --- - - # Custom SCons installation stuff. - if Options.hardlink_scons: - create_basename_script = self.hardlink_scons - elif Options.symlink_scons: - create_basename_script = self.symlink_scons - elif Options.install_scons_script: - create_basename_script = self.copy_scons - else: - create_basename_script = self.do_nothing - - if Options.install_version_script: - create_version_script = self.copy_scons - else: - create_version_script = self.do_nothing - - inputs = self.get_inputs() - bat_scripts = [x for x in inputs if x[-4:] == '.bat'] - non_bat_scripts = [x for x in inputs if x[-4:] != '.bat'] - - self.outfiles = [] - self.mkpath(self.install_dir) - - for src in non_bat_scripts: - base = os.path.basename(src) - scons = os.path.join(self.install_dir, base) - scons_ver = scons + '-' + Version - if is_win32: - scons = scons + '.py' - scons_ver = scons_ver + '.py' - create_version_script(src, scons_ver) - create_basename_script(src, scons, scons_ver) - - if Options.install_bat: - if is_win32: - bat_install_dir = get_scons_prefix(self.install_dir, is_win32) - else: - bat_install_dir = self.install_dir - for src in bat_scripts: - scons_bat = os.path.join(bat_install_dir, 'scons.bat') - scons_version_bat = os.path.join(bat_install_dir, - 'scons-' + Version + '.bat') - self.copy_scons(src, scons_bat) - self.copy_scons(src, scons_version_bat) - - # --- distutils copy/paste --- - if hasattr(os, 'chmod') and hasattr(os, 'stat'): - # Set the executable bits (owner, group, and world) on - # all the scripts we just installed. - for file in self.get_outputs(): - if self.dry_run: - # log.info("changing mode of %s", file) - pass - else: - # Use symbolic versions of permissions so this script doesn't fail to parse under python3.x - exec_and_read_permission = stat.S_IXOTH | stat.S_IXUSR | stat.S_IXGRP | stat.S_IROTH | stat.S_IRUSR | stat.S_IRGRP - mode_mask = 4095 # Octal 07777 used because python3 has different octal syntax than python 2 - mode = ((os.stat(file)[stat.ST_MODE]) | - exec_and_read_permission) & mode_mask - # log.info("changing mode of %s to %o", file, mode) - os.chmod(file, mode) - # --- /distutils copy/paste --- - - -class build_scripts(_build_scripts): - def finalize_options(self): - _build_scripts.finalize_options(self) - self.build_dir = os.path.join('build', 'scripts') - - -class install_data(_install_data): - def initialize_options(self): - _install_data.initialize_options(self) - - def finalize_options(self): - _install_data.finalize_options(self) - if force_to_usr_local(self): - self.install_dir = '/usr/local' - if Options.install_man: - if is_win32: - dir = 'Doc' - else: - dir = os.path.join('man', 'man1') - self.data_files = [(dir, man_pages)] - man_dir = os.path.join(self.install_dir, dir) - msg = "Installed SCons man pages into %s" % man_dir - Installed.append(msg) - else: - self.data_files = [] - - -description = "Open Source next-generation build tool." - -long_description = """Open Source next-generation build tool. -Improved, cross-platform substitute for the classic Make -utility. In short, SCons is an easier, more reliable -and faster way to build software.""" - -scripts = [ - 'script/scons', - 'script/sconsign', - 'script/scons-time', - 'script/scons-configure-cache', - - # We include scons.bat in the list of scripts, even on UNIX systems, - # because we provide an option to allow it be installed explicitly, - # for example if you're installing from UNIX on a share that's - # accessible to Windows and you want the scons.bat. - 'script/scons.bat', -] - -arguments = { - 'name': "scons", - 'version': Version, - 'description': description, - 'long_description': long_description, - 'author': 'William Deegan', - 'author_email': 'bill@baddogconsulting.com', - 'url': "http://www.scons.org/", - 'packages': ["SCons", - "SCons.compat", - "SCons.Node", - "SCons.Platform", - "SCons.Scanner", - "SCons.Script", - "SCons.Tool", - "SCons.Tool.clangCommon", - "SCons.Tool.docbook", - 'SCons.Tool.clangCommon', - "SCons.Tool.MSCommon", - "SCons.Tool.packaging", - "SCons.Variables", - ], - 'package_dir': {'': 'engine', - 'SCons.Tool.docbook': 'engine/SCons/Tool/docbook'}, - 'package_data': {'SCons.Tool.docbook': ['docbook-xsl-1.76.1/*', - 'docbook-xsl-1.76.1/common/*', - 'docbook-xsl-1.76.1/docsrc/*', - 'docbook-xsl-1.76.1/eclipse/*', - 'docbook-xsl-1.76.1/epub/*', - 'docbook-xsl-1.76.1/epub/bin/*', - 'docbook-xsl-1.76.1/epub/bin/lib/*', - 'docbook-xsl-1.76.1/epub/bin/xslt/*', - 'docbook-xsl-1.76.1/extensions/*', - 'docbook-xsl-1.76.1/fo/*', - 'docbook-xsl-1.76.1/highlighting/*', - 'docbook-xsl-1.76.1/html/*', - 'docbook-xsl-1.76.1/htmlhelp/*', - 'docbook-xsl-1.76.1/images/*', - 'docbook-xsl-1.76.1/images/callouts/*', - 'docbook-xsl-1.76.1/images/colorsvg/*', - 'docbook-xsl-1.76.1/javahelp/*', - 'docbook-xsl-1.76.1/lib/*', - 'docbook-xsl-1.76.1/manpages/*', - 'docbook-xsl-1.76.1/params/*', - 'docbook-xsl-1.76.1/profiling/*', - 'docbook-xsl-1.76.1/roundtrip/*', - 'docbook-xsl-1.76.1/slides/browser/*', - 'docbook-xsl-1.76.1/slides/fo/*', - 'docbook-xsl-1.76.1/slides/graphics/*', - 'docbook-xsl-1.76.1/slides/graphics/active/*', - 'docbook-xsl-1.76.1/slides/graphics/inactive/*', - 'docbook-xsl-1.76.1/slides/graphics/toc/*', - 'docbook-xsl-1.76.1/slides/html/*', - 'docbook-xsl-1.76.1/slides/htmlhelp/*', - 'docbook-xsl-1.76.1/slides/keynote/*', - 'docbook-xsl-1.76.1/slides/keynote/xsltsl/*', - 'docbook-xsl-1.76.1/slides/svg/*', - 'docbook-xsl-1.76.1/slides/xhtml/*', - 'docbook-xsl-1.76.1/template/*', - 'docbook-xsl-1.76.1/tests/*', - 'docbook-xsl-1.76.1/tools/bin/*', - 'docbook-xsl-1.76.1/tools/make/*', - 'docbook-xsl-1.76.1/webhelp/*', - 'docbook-xsl-1.76.1/webhelp/docs/*', - 'docbook-xsl-1.76.1/webhelp/docs/common/*', - 'docbook-xsl-1.76.1/webhelp/docs/common/css/*', - 'docbook-xsl-1.76.1/webhelp/docs/common/images/*', - 'docbook-xsl-1.76.1/webhelp/docs/common/jquery/*', - 'docbook-xsl-1.76.1/webhelp/docs/common/jquery/theme-redmond/*', - 'docbook-xsl-1.76.1/webhelp/docs/common/jquery/theme-redmond/images/*', - 'docbook-xsl-1.76.1/webhelp/docs/common/jquery/treeview/*', - 'docbook-xsl-1.76.1/webhelp/docs/common/jquery/treeview/images/*', - 'docbook-xsl-1.76.1/webhelp/docs/content/*', - 'docbook-xsl-1.76.1/webhelp/docs/content/search/*', - 'docbook-xsl-1.76.1/webhelp/docs/content/search/stemmers/*', - 'docbook-xsl-1.76.1/webhelp/docsrc/*', - 'docbook-xsl-1.76.1/webhelp/template/*', - 'docbook-xsl-1.76.1/webhelp/template/common/*', - 'docbook-xsl-1.76.1/webhelp/template/common/css/*', - 'docbook-xsl-1.76.1/webhelp/template/common/images/*', - 'docbook-xsl-1.76.1/webhelp/template/common/jquery/*', - 'docbook-xsl-1.76.1/webhelp/template/common/jquery/theme-redmond/*', - 'docbook-xsl-1.76.1/webhelp/template/common/jquery/theme-redmond/images/*', - 'docbook-xsl-1.76.1/webhelp/template/common/jquery/treeview/*', - 'docbook-xsl-1.76.1/webhelp/template/common/jquery/treeview/images/*', - 'docbook-xsl-1.76.1/webhelp/template/content/search/*', - 'docbook-xsl-1.76.1/webhelp/template/content/search/stemmers/*', - 'docbook-xsl-1.76.1/webhelp/xsl/*', - 'docbook-xsl-1.76.1/website/*', - 'docbook-xsl-1.76.1/xhtml/*', - 'docbook-xsl-1.76.1/xhtml-1_1/*', - 'utils/*']}, - 'data_files': [('man/man1', man_pages)], - 'scripts': scripts, - 'cmdclass': {'install': install, - 'install_lib': install_lib, - 'install_data': install_data, - 'install_scripts': install_scripts, - 'build_scripts': build_scripts}, - 'install_requires' : { - "pywin32;platform_system=='Windows'" - } -} - -distutils.core.setup(**arguments) - -if Installed: - for i in Installed: - print(i) - -# Local Variables: -# tab-width:4 -# indent-tabs-mode:nil -# End: -# vim: set expandtab tabstop=4 shiftwidth=4: -- cgit v0.12 From e3908345f0c0aef30cc332995384a1fa310ba46c Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 30 Mar 2020 16:24:30 -0700 Subject: Update location of LICENSE and use same version in sdist and wheel --- MANIFEST.in | 5 +++++ setup.cfg | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 04aee14..c283957 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,4 +2,9 @@ global-exclude **Tests.py recursive-include src/engine/SCons/Tool/docbook * +# For license file +include LICENSE + + + diff --git a/setup.cfg b/setup.cfg index 2eb7728..614303c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,13 +1,13 @@ [metadata] name = SCons -version=3.9.9a99 +version=3.9.9a991 license = MIT author = William Deegan author_email =bill@baddogconsulting.com long_description = file: README.rst description = Open Source next-generation build tool. group = Development/Tools -license_file = src/LICENSE.txt +license_file = LICENSE url = http://www.scons.org/ -- cgit v0.12 From 8bdb821413e97e614311999006409f6702f91610 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 30 Mar 2020 16:25:05 -0700 Subject: Remove src/LICENSE.txt. Just use the version in the basedir --- src/LICENSE.txt | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 src/LICENSE.txt diff --git a/src/LICENSE.txt b/src/LICENSE.txt deleted file mode 100644 index 1641aba..0000000 --- a/src/LICENSE.txt +++ /dev/null @@ -1,22 +0,0 @@ -MIT License - -__COPYRIGHT__ - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be included -in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -- cgit v0.12 From a369a98389636358108827a3de04fa17970bf6bb Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 30 Mar 2020 16:54:32 -0700 Subject: Clean up unused imports --- SConstruct | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/SConstruct b/SConstruct index a76471f..5be7d22 100644 --- a/SConstruct +++ b/SConstruct @@ -31,13 +31,10 @@ month_year = 'December 2019' # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # -import fnmatch -import os import os.path import sys import textwrap -import bootstrap project = 'scons' default_version = '3.1.2' @@ -64,6 +61,7 @@ for a in addpaths: if a not in sys.path: sys.path.append(a) +# Use site_scons logic to process command line arguments command_line = BuildCommandLine(default_version) command_line.process_command_line_vars() -- cgit v0.12 From 90657e02371290930ddcedcedbc9494d888ea670 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 30 Mar 2020 16:56:13 -0700 Subject: remove unused imports. --- bin/scons-proc.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/bin/scons-proc.py b/bin/scons-proc.py index 6c1f66a..e039cd0 100644 --- a/bin/scons-proc.py +++ b/bin/scons-proc.py @@ -11,10 +11,7 @@ # import getopt import os -import re -import string import sys -from io import StringIO # usable as of 2.6; takes unicode only import SConsDoc from SConsDoc import tf as stf @@ -107,10 +104,10 @@ class SCons_XML(object): for k, v in kw.items(): setattr(self, k, v) - def fopen(self, name): + def fopen(self, name, mode='w'): if name == '-': return sys.stdout - return open(name, 'w') + return open(name, mode) def write(self, files): gen, mod = files.split(',') -- cgit v0.12 From a9ab8cae92f95e7410cb534020ff03e15c753f9e Mon Sep 17 00:00:00 2001 From: William Deegan Date: Thu, 9 Apr 2020 14:01:07 -0700 Subject: Update CHANGES.txt --- src/CHANGES.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/CHANGES.txt b/src/CHANGES.txt index 6ec16f3..ac8a3ac 100755 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -30,6 +30,8 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Add msys2 installed mingw default path to PATH for mingw tool. - C:\msys64\mingw64\bin - Purge obsolete internal build and tooling scripts + - Resolve Issue #3451 and Issue #3450 - Rewrite SCons setup.py and packaging. Move script logic to entry points so + package can create scripts which use the correct version of Python. From Jeremy Elson: - Updated design doc to use the correct syntax for Depends() -- cgit v0.12 From a4c2ee46188f8b43a451af01eda8b3cfefb1b663 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Fri, 10 Apr 2020 12:24:03 -0600 Subject: Second try at fixing sets/uses Tool doc generation. [ci skip] Fixes #3580 a different way: generate the link itself into the tools.gen file, instead of post-processing the broken entity reference back into a usable form. Signed-off-by: Mats Wichmann --- bin/docs-update-generated.py | 10 +++------- bin/scons-proc.py | 28 ++++++++++++++++++---------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/bin/docs-update-generated.py b/bin/docs-update-generated.py index 98c923d..0a373aa 100644 --- a/bin/docs-update-generated.py +++ b/bin/docs-update-generated.py @@ -51,13 +51,9 @@ def generate_all(): '-v', argpair('variables')] + flist, shell=False) - cp.check_returncode() # bail if it failed - # lxml: fixup possibly broken tools.gen: - with open(os.path.join(gen_folder, 'tools.gen'), 'r') as f : - filedata = f.read() - filedata = filedata.replace(r'&cv-link', r'&cv-link') - with open(os.path.join(gen_folder, 'tools.gen'), 'w') as f : - f.write(filedata) + # No-op: scons-proc doesn't actually set an exit code at the moment. + if cp.returncode: + print("Generation failed", file=sys.stderr) if __name__ == "__main__": diff --git a/bin/scons-proc.py b/bin/scons-proc.py index 6c1f66a..ac52894 100644 --- a/bin/scons-proc.py +++ b/bin/scons-proc.py @@ -141,23 +141,31 @@ class SCons_XML(object): added = True stf.appendNode(vl, stf.copyNode(s)) + # Generate the text for sets/uses lists of construction vars. + # This used to include an entity reference which would be replaced + # by the link to the cvar, but with lxml, dumping out the tree + # with tostring() will encode the & introducing the entity, + # breaking it. Instead generate the actual link. (issue #3580) if v.sets: added = True vp = stf.newNode("para") - # if using lxml, the &entity; entries will be encoded, - # effectively breaking them. should fix, - # for now handled post-process in calling script. - s = ['&cv-link-%s;' % x for x in v.sets] - stf.setText(vp, 'Sets: ' + ', '.join(s) + '.') + stf.setText(vp, "Sets: ") + for setv in v.sets: + link = stf.newSubNode(vp, "link", linkend="cv-%s" % setv) + linktgt = stf.newSubNode(link, "varname") + stf.setText(linktgt, "$" + setv) + stf.setTail(link, " ") stf.appendNode(vl, vp) + if v.uses: added = True vp = stf.newNode("para") - # if using lxml, the &entity; entries will be encoded, - # effectively breaking them. should fix, - # for now handled post-process in calling script. - u = ['&cv-link-%s;' % x for x in v.uses] - stf.setText(vp, 'Uses: ' + ', '.join(u) + '.') + stf.setText(vp, "Uses: ") + for use in v.uses: + link = stf.newSubNode(vp, "link", linkend="cv-%s" % use) + linktgt = stf.newSubNode(link, "varname") + stf.setText(linktgt, "$" + use) + stf.setTail(link, " ") stf.appendNode(vl, vp) # Still nothing added to this list item? -- cgit v0.12 From 331107e44adf317090693650afd534c11d325a4a Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sat, 11 Apr 2020 12:52:43 -0600 Subject: Update doc wording for IMPLICIT_COMMAND_DEPEDENCIES [ci skip] Reword, remove a bit of redundancy by describing actions containing && in one place instead of repeatedly. Actually mention the case of setting the cvar to True. Doc-only change. Signed-off-by: Mats Wichmann --- src/engine/SCons/Action.xml | 91 ++++++++++++++++++++++++++------------------- 1 file changed, 53 insertions(+), 38 deletions(-) diff --git a/src/engine/SCons/Action.xml b/src/engine/SCons/Action.xml index 350082d..107e389 100644 --- a/src/engine/SCons/Action.xml +++ b/src/engine/SCons/Action.xml @@ -35,19 +35,39 @@ executed to build targets. By default, SCons will add to each target an implicit dependency on the command -represented by the first argument on any -command line it executes. +represented by the first argument of any +command line it executes (which is typically +the command itself). By setting such +a dependency, &SCons; can determine that +a target should be rebuilt if the command changes, +such as when a compiler is upgraded to a new version. The specific file for the dependency is found by searching the PATH variable in the -ENV -environment used to execute the command. +ENV dictionary +in the &consenv; used to execute the command. +The default is the same as +setting the &consvar; +&cv-IMPLICIT_COMMAND_DEPENDENCIES; +to a true value (True, +any non-zero number, etc.). -If the construction variable -&cv-IMPLICIT_COMMAND_DEPENDENCIES; +Action strings can be segmented by the +use of an AND operator, &&. +In a segemented string, each segment is a separate +command line, these are run +sequentially until one fails or the entire +sequence has been executed. If an +action string is segmented, then the selected +behavior of &cv-IMPLICIT_COMMAND_DEPENDENCIES; +is applied to each segment. + + + +If &cv-IMPLICIT_COMMAND_DEPENDENCIES; is set to a false value (None, False, @@ -55,50 +75,45 @@ is set to a false value etc.), then the implicit dependency will not be added to the targets -built with that construction environment. +built with that &consenv;. -If the construction variable -&cv-IMPLICIT_COMMAND_DEPENDENCIES; -is set to 2 or higher, -then that number of entries in the command -string will be scanned for relative or absolute -paths. The count will reset after any -&& entries are found. -The first command in the action string and -the first after any && -entries will be found by searching the -PATH variable in the -ENV environment used to -execute the command. All other commands will -only be found if they are absolute paths or -valid paths relative to the working directory. +If &cv-IMPLICIT_COMMAND_DEPENDENCIES; +is set to integer 2 or higher, +then that number of arguments in the command line +will be scanned for relative or absolute paths. +If any are present, they will be added as +implicit dependencies to the targets built +with that &consenv;. +The first argument in the command line will be +searched for using the PATH +variable in the ENV dictionary +in the &consenv; used to execute the command. +The other arguments will only be found if they +are absolute paths or valid paths relative +to the working directory. -If the construction variable -&cv-IMPLICIT_COMMAND_DEPENDENCIES; -is set to all, then -all entries in all command strings will be -scanned for relative or absolute paths. If -any are present, they will be added as +If &cv-IMPLICIT_COMMAND_DEPENDENCIES; +is set to all, +then all arguments in the command line will be +scanned for relative or absolute paths. +If any are present, they will be added as implicit dependencies to the targets built -with that construction environment. -not be added to the targets built with that -construction environment. The first command -in the action string and the first after any -&& entries will be found -by searching the PATH -variable in the ENV -environment used to execute the command. -All other commands will only be found if they +with that &consenv;. +The first argument in the command line will be +searched for using the PATH +variable in the ENV dictionary +in the &consenv; used to execute the command. +The other arguments will only be found if they are absolute paths or valid paths relative to the working directory. -env = Environment(IMPLICIT_COMMAND_DEPENDENCIES = 0) +env = Environment(IMPLICIT_COMMAND_DEPENDENCIES=False) -- cgit v0.12 From 7819d4c757bba420ec4b96251c188f7665a49369 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 11 Apr 2020 16:59:42 -0700 Subject: Address github review notes and sider issues --- SConstruct | 2 +- bin/SConsDoc.py | 3 +- bin/scons-time.py | 180 +++++++++++++++++---------------- runtest.py | 22 +--- setup.py | 2 +- site_scons/BuildCommandLine.py | 9 +- site_scons/epydoc.py | 2 +- site_scons/site_init.py | 4 +- src/engine/SCons/Utilities/sconsign.py | 10 +- test/scons-time/run/archive/dir.py | 1 - 10 files changed, 116 insertions(+), 119 deletions(-) diff --git a/SConstruct b/SConstruct index 5be7d22..4fdb401 100644 --- a/SConstruct +++ b/SConstruct @@ -37,7 +37,7 @@ import textwrap project = 'scons' -default_version = '3.1.2' +default_version = '3.9.9' copyright = "Copyright (c) %s The SCons Foundation" % copyright_years # diff --git a/bin/SConsDoc.py b/bin/SConsDoc.py index 1e88c5b..e211b3c 100644 --- a/bin/SConsDoc.py +++ b/bin/SConsDoc.py @@ -114,6 +114,7 @@ import os.path import re import sys import copy +import importlib # Do we have libxml2/libxslt/lxml? has_libxml2 = True @@ -867,7 +868,7 @@ def importfile(path): try: return importlib._bootstrap._load(spec) except ImportError: - raise ErrorDuringImport(path, sys.exc_info()) + raise Exception(path, sys.exc_info()) # Local Variables: # tab-width:4 diff --git a/bin/scons-time.py b/bin/scons-time.py index 6494349..37de2f0 100644 --- a/bin/scons-time.py +++ b/bin/scons-time.py @@ -40,7 +40,7 @@ import shutil import sys import tempfile import time -import subprocess + def HACK_for_exec(cmd, *args): """ @@ -49,9 +49,13 @@ def HACK_for_exec(cmd, *args): This function is a hack that calls exec() in a function with no internal functions. """ - if not args: exec(cmd) - elif len(args) == 1: exec(cmd, args[0]) - else: exec(cmd, args[0], args[1]) + if not args: + exec(cmd) + elif len(args) == 1: + exec(cmd, args[0]) + else: + exec(cmd, args[0], args[1]) + class Plotter(object): def increment_size(self, largest): @@ -76,6 +80,7 @@ class Plotter(object): increment = self.increment_size(largest) return ((largest + increment - 1) // increment) * increment + class Line(object): def __init__(self, points, type, title, label, comment, fmt="%s %s"): self.points = points @@ -111,10 +116,11 @@ class Line(object): print('e') def get_x_values(self): - return [ p[0] for p in self.points ] + return [p[0] for p in self.points] def get_y_values(self): - return [ p[1] for p in self.points ] + return [p[1] for p in self.points] + class Gnuplotter(Plotter): @@ -201,22 +207,21 @@ class Gnuplotter(Plotter): max_y = self.max_graph_value(self.get_max_y()) incr = (max_y - min_y) / 10.0 start = min_y + (max_y / 2.0) + (2.0 * incr) - position = [ start - (i * incr) for i in range(5) ] + position = [start - (i * incr) for i in range(5)] inx = 1 for line in self.lines: - line.print_label(inx, line.points[0][0]-1, - position[(inx-1) % len(position)]) + line.print_label(inx, line.points[0][0] - 1, + position[(inx - 1) % len(position)]) inx += 1 - plot_strings = [ self.plot_string(l) for l in self.lines ] + plot_strings = [self.plot_string(l) for l in self.lines] print('plot ' + ', \\\n '.join(plot_strings)) for line in self.lines: line.print_points() - def untar(fname): import tarfile tar = tarfile.open(name=fname, mode='r') @@ -224,6 +229,7 @@ def untar(fname): tar.extract(tarinfo) tar.close() + def unzip(fname): import zipfile zf = zipfile.ZipFile(fname, 'r') @@ -231,11 +237,12 @@ def unzip(fname): dir = os.path.dirname(name) try: os.makedirs(dir) - except: + except OSError: pass with open(name, 'wb') as f: f.write(zf.read(name)) + def read_tree(dir): for dirpath, dirnames, filenames in os.walk(dir): for fn in filenames: @@ -244,14 +251,15 @@ def read_tree(dir): with open(fn, 'rb') as f: f.read() + def redirect_to_file(command, log): return '%s > %s 2>&1' % (command, log) + def tee_to_file(command, log): return '%s 2>&1 | tee %s' % (command, log) - class SConsTimer(object): """ Usage: scons-time SUBCOMMAND [ARGUMENTS] @@ -267,45 +275,45 @@ class SConsTimer(object): """ name = 'scons-time' - name_spaces = ' '*len(name) + name_spaces = ' ' * len(name) def makedict(**kw): return kw default_settings = makedict( - chdir = None, - config_file = None, - initial_commands = [], - key_location = 'bottom left', - orig_cwd = os.getcwd(), - outdir = None, - prefix = '', - python = '"%s"' % sys.executable, - redirect = redirect_to_file, - scons = None, - scons_flags = '--debug=count --debug=memory --debug=time --debug=memoizer', - scons_lib_dir = None, - scons_wrapper = None, - startup_targets = '--help', - subdir = None, - subversion_url = None, - svn = 'svn', - svn_co_flag = '-q', - tar = 'tar', - targets = '', - targets0 = None, - targets1 = None, - targets2 = None, - title = None, - unzip = 'unzip', - verbose = False, - vertical_bars = [], - - unpack_map = { - '.tar.gz' : (untar, '%(tar)s xzf %%s'), - '.tgz' : (untar, '%(tar)s xzf %%s'), - '.tar' : (untar, '%(tar)s xf %%s'), - '.zip' : (unzip, '%(unzip)s %%s'), + chdir=None, + config_file=None, + initial_commands=[], + key_location='bottom left', + orig_cwd=os.getcwd(), + outdir=None, + prefix='', + python='"%s"' % sys.executable, + redirect=redirect_to_file, + scons=None, + scons_flags='--debug=count --debug=memory --debug=time --debug=memoizer', + scons_lib_dir=None, + scons_wrapper=None, + startup_targets='--help', + subdir=None, + subversion_url=None, + svn='svn', + svn_co_flag='-q', + tar='tar', + targets='', + targets0=None, + targets1=None, + targets2=None, + title=None, + unzip='unzip', + verbose=False, + vertical_bars=[], + + unpack_map={ + '.tar.gz': (untar, '%(tar)s xzf %%s'), + '.tgz': (untar, '%(tar)s xzf %%s'), + '.tar': (untar, '%(tar)s xf %%s'), + '.zip': (unzip, '%(unzip)s %%s'), }, ) @@ -329,10 +337,10 @@ class SConsTimer(object): ] stage_strings = { - 'pre-read' : 'Memory before reading SConscript files:', - 'post-read' : 'Memory after reading SConscript files:', - 'pre-build' : 'Memory before building targets:', - 'post-build' : 'Memory after building targets:', + 'pre-read': 'Memory before reading SConscript files:', + 'post-read': 'Memory after reading SConscript files:', + 'pre-build': 'Memory before building targets:', + 'post-build': 'Memory after building targets:', } memory_string_all = 'Memory ' @@ -340,10 +348,10 @@ class SConsTimer(object): default_stage = stages[-1] time_strings = { - 'total' : 'Total build time', - 'SConscripts' : 'Total SConscript file execution time', - 'SCons' : 'Total SCons execution time', - 'commands' : 'Total command execution time', + 'total': 'Total build time', + 'SConscripts': 'Total SConscript file execution time', + 'SCons': 'Total SCons execution time', + 'commands': 'Total command execution time', } time_string_all = 'Total .* time' @@ -425,7 +433,7 @@ class SConsTimer(object): Executes a list of commands, substituting values from the specified dictionary. """ - commands = [ self.subst_variables(c, dict) for c in commands ] + commands = [self.subst_variables(c, dict) for c in commands] for action, string, args in commands: self.display(string, *args) sys.stdout.flush() @@ -444,11 +452,11 @@ class SConsTimer(object): p = os.popen(command) output = p.read() p.close() - #TODO: convert to subrocess, os.popen is obsolete. This didn't work: - #process = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True) - #output = process.stdout.read() - #process.stdout.close() - #process.wait() + # TODO: convert to subrocess, os.popen is obsolete. This didn't work: + # process = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True) + # output = process.stdout.read() + # process.stdout.close() + # process.wait() if self.verbose: sys.stdout.write(output) # TODO: Figure out @@ -563,7 +571,7 @@ class SConsTimer(object): except IndexError: t = '??? %s ???' % i results[i].sort() - gp.line(results[i], i+1, t, None, t, fmt=fmt) + gp.line(results[i], i + 1, t, None, t, fmt=fmt) for bar_tuple in self.vertical_bars: try: @@ -592,11 +600,13 @@ class SConsTimer(object): if lines[0] == '': lines = lines[1:] spaces = re.match(' *', lines[0]).group(0) - def strip_initial_spaces(l, s=spaces): - if l.startswith(spaces): - l = l[len(spaces):] - return l - return '\n'.join([ strip_initial_spaces(l) for l in lines ]) + '\n' + + def strip_initial_spaces(line, s=spaces): + if line.startswith(spaces): + line = line[len(spaces):] + return line + + return '\n'.join([strip_initial_spaces(l) for l in lines]) + '\n' def profile_name(self, invocation): """ @@ -625,7 +635,7 @@ class SConsTimer(object): sys.stderr.write('file %s has no contents!\n' % repr(file)) return None result = re.findall(r'%s: ([\d.]*)' % search_string, contents)[-4:] - result = [ float(r) for r in result ] + result = [float(r) for r in result] if time_string is not None: try: result = result[0] @@ -646,7 +656,7 @@ class SConsTimer(object): sys.stderr.write('%s Cannot use the "func" subcommand.\n' % self.name_spaces) sys.exit(1) statistics = pstats.Stats(file).stats - matches = [ e for e in statistics.items() if e[0][2] == function ] + matches = [e for e in statistics.items() if e[0][2] == function] r = matches[0] return r[0][0], r[0][1], r[0][2], r[1][3] @@ -667,8 +677,8 @@ class SConsTimer(object): search_string = memory_string with open(file) as f: lines = f.readlines() - lines = [ l for l in lines if l.startswith(search_string) ][-4:] - result = [ int(l.split()[-1]) for l in lines[-4:] ] + lines = [l for l in lines if l.startswith(search_string)][-4:] + result = [int(l.split()[-1]) for l in lines[-4:]] if len(result) == 1: result = result[0] return result @@ -680,13 +690,12 @@ class SConsTimer(object): object_string = ' ' + object_name + '\n' with open(file) as f: lines = f.readlines() - line = [ l for l in lines if l.endswith(object_string) ][0] - result = [ int(field) for field in line.split()[:4] ] + line = [l for l in lines if l.endswith(object_string)][0] + result = [int(field) for field in line.split()[:4]] if index is not None: result = result[index] return result - command_alias = {} def execute_subcommand(self, argv): @@ -841,7 +850,7 @@ class SConsTimer(object): for file in args: try: f, line, func, time = \ - self.get_function_profile(file, function_name) + self.get_function_profile(file, function_name) except ValueError as e: sys.stderr.write("%s: func: %s: %s\n" % (self.name, file, e)) @@ -956,7 +965,7 @@ class SConsTimer(object): args = self.args_to_files(args, tail) - cwd_ = os.getcwd() + os.sep + # cwd_ = os.getcwd() + os.sep if format == 'ascii': @@ -1216,11 +1225,11 @@ class SConsTimer(object): except ValueError: result.append(int(n)) else: - result.extend(list(range(int(x), int(y)+1))) + result.extend(list(range(int(x), int(y) + 1))) return result def scons_path(self, dir): - return os.path.join(dir,'scripts', 'scons.py') + return os.path.join(dir, 'scripts', 'scons.py') def scons_lib_dir_path(self, dir): return os.path.join(dir, 'src', 'engine') @@ -1247,9 +1256,9 @@ class SConsTimer(object): if prepare: prepare(commands, removals) - save_scons = self.scons - save_scons_wrapper = self.scons_wrapper - save_scons_lib_dir = self.scons_lib_dir + save_scons = self.scons + save_scons_wrapper = self.scons_wrapper + save_scons_lib_dir = self.scons_lib_dir if self.outdir is None: self.outdir = self.orig_cwd @@ -1308,7 +1317,7 @@ class SConsTimer(object): commands.extend([ (lambda: read_tree('.'), - 'find * -type f | xargs cat > /dev/null'), + 'find * -type f | xargs cat > /dev/null'), (self.set_env, 'export %%s=%%s', 'SCONS_LIB_DIR', self.scons_lib_dir), @@ -1338,9 +1347,9 @@ class SConsTimer(object): self.run_command_list(commands, self.__dict__) - self.scons = save_scons - self.scons_lib_dir = save_scons_lib_dir - self.scons_wrapper = save_scons_wrapper + self.scons = save_scons + self.scons_lib_dir = save_scons_lib_dir + self.scons_wrapper = save_scons_wrapper # @@ -1454,6 +1463,7 @@ class SConsTimer(object): sys.stderr.write('%s: time: Unknown format "%s".\n' % (self.name, format)) sys.exit(1) + if __name__ == '__main__': opts, args = getopt.getopt(sys.argv[1:], 'h?V', ['help', 'version']) diff --git a/runtest.py b/runtest.py index 10d16ba..519cc74 100755 --- a/runtest.py +++ b/runtest.py @@ -486,26 +486,8 @@ ld = None if not baseline or baseline == '.': base = cwd elif baseline == '-': - url = None - with os.popen("svn info 2>&1", "r") as p: - svn_info = p.read() - match = re.search(r'URL: (.*)', svn_info) - if match: - url = match.group(1) - if not url: - sys.stderr.write('runtest.py: could not find a URL:\n') - sys.stderr.write(svn_info) - sys.exit(1) - import tempfile - base = tempfile.mkdtemp(prefix='runtest-tmp-') - - command = 'cd %s && svn co -q %s' % (base, url) - - base = os.path.join(base, os.path.split(url)[1]) - if printcommand: - print(command) - if execute_tests: - os.system(command) + print("This logic used to checkout from svn. It's been removed. If you used this, please let us know on devel mailing list, IRC, or discord server") + sys.exit(-1) else: base = baseline diff --git a/setup.py b/setup.py index 349a24e..d6b7783 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ import fnmatch -from setuptools import find_packages, setup +from setuptools import setup from setuptools.command.build_py import build_py as build_py_orig diff --git a/site_scons/BuildCommandLine.py b/site_scons/BuildCommandLine.py index 4694472..b8c7d63 100644 --- a/site_scons/BuildCommandLine.py +++ b/site_scons/BuildCommandLine.py @@ -102,25 +102,28 @@ class BuildCommandLine(object): self.revision = ARGUMENTS.get('REVISION', '') - def generate_build_id(revision): + def _generate_build_id(revision): return revision + generate_build_id=_generate_build_id + if not self.revision and BuildCommandLine.git: with os.popen("%s rev-parse HEAD 2> /dev/null" % BuildCommandLine.git, "r") as p: self.git_hash = p.read().strip() - def generate_build_id(revision): + def _generate_build_id_git(revision): result = self.git_hash if [l for l in self.git_status_lines if 'modified' in l]: result = result + '[MODIFIED]' return result + generate_build_id = _generate_build_id_git self.revision = self.git_hash self.checkpoint = ARGUMENTS.get('CHECKPOINT', '') if self.checkpoint: if self.checkpoint == 'd': - cself.heckpoint = time.strftime('%Y%m%d', time.localtime(time.time())) + self.checkpoint = time.strftime('%Y%m%d', time.localtime(time.time())) elif self.checkpoint == 'r': self.checkpoint = 'r' + self.revision self.version = self.version + '.beta.' + self.checkpoint diff --git a/site_scons/epydoc.py b/site_scons/epydoc.py index da74d9c..3d6e5de 100644 --- a/site_scons/epydoc.py +++ b/site_scons/epydoc.py @@ -53,7 +53,7 @@ if not epydoc_cli: from epydoc.docbuilder import build_doc_index from epydoc.docwriter.html import HTMLWriter - from epydoc.docwriter.latex import LatexWriter + # from epydoc.docwriter.latex import LatexWriter # first arg is a list where can be names of python package dirs, # python files, object names or objects itself diff --git a/site_scons/site_init.py b/site_scons/site_init.py index 7e7c569..81c4753 100644 --- a/site_scons/site_init.py +++ b/site_scons/site_init.py @@ -1,8 +1,10 @@ +import os.path + from SConsRevision import SCons_revision from Utilities import is_windows, whereis, platform, deb_date from zip_utils import unzipit, zipit, zcat from soe_utils import soelim, soscan, soelimbuilder -from epydoc import epydoc_cli, epydoc_commands +# from epydoc import epydoc_cli, epydoc_commands from BuildCommandLine import BuildCommandLine gzip = whereis('gzip') diff --git a/src/engine/SCons/Utilities/sconsign.py b/src/engine/SCons/Utilities/sconsign.py index 8b16b18..89084ba 100644 --- a/src/engine/SCons/Utilities/sconsign.py +++ b/src/engine/SCons/Utilities/sconsign.py @@ -218,12 +218,12 @@ def nodeinfo_raw(name, ninfo, prefix=""): keys = ninfo.field_list + ['_version_id'] except AttributeError: keys = sorted(d.keys()) - l = [] - for k in keys: - l.append('%s: %s' % (repr(k), repr(d.get(k)))) + values = [] + for key in keys: + values.append('%s: %s' % (repr(key), repr(d.get(key)))) if '\n' in name: name = repr(name) - return name + ': {' + ', '.join(l) + '}' + return name + ': {' + ', '.join(values) + '}' def nodeinfo_cooked(name, ninfo, prefix=""): @@ -280,7 +280,7 @@ def printentries(entries, location): for name in sorted(entries.keys()): entry = entries[name] try: - ninfo = entry.ninfo + entry.ninfo except AttributeError: print(name + ":") else: diff --git a/test/scons-time/run/archive/dir.py b/test/scons-time/run/archive/dir.py index 6b6d992..590d568 100644 --- a/test/scons-time/run/archive/dir.py +++ b/test/scons-time/run/archive/dir.py @@ -32,7 +32,6 @@ directory tree. import TestSCons_time test = TestSCons_time.TestSCons_time() -test.verbose_set(1) test.write_fake_scons_py() -- cgit v0.12 From e0ad3d9278e9344c26557265f9970ded675bb92d Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 11 Apr 2020 21:24:47 -0700 Subject: resolve some sider issues --- bin/scons-time.py | 40 +++++++++++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/bin/scons-time.py b/bin/scons-time.py index 37de2f0..693b254 100644 --- a/bin/scons-time.py +++ b/bin/scons-time.py @@ -894,7 +894,11 @@ class SConsTimer(object): def do_mem(self, argv): format = 'ascii' - logfile_path = lambda x: x + def _logfile_path(x): + return x + + logfile_path = _logfile_path + stage = self.default_stage tail = None @@ -943,10 +947,12 @@ class SConsTimer(object): if self.chdir: os.chdir(self.chdir) - logfile_path = lambda x: os.path.join(self.chdir, x) + def _logfile_path_join(x): + return os.path.join(self.chdir, x) - if not args: + logfile_path = _logfile_path_join + if not args: pattern = '%s*.log' % self.prefix args = self.args_to_files([pattern], tail) @@ -1008,7 +1014,12 @@ class SConsTimer(object): def do_obj(self, argv): format = 'ascii' - logfile_path = lambda x: x + + def _logfile_path(x): + return x + + logfile_path = _logfile_path + stage = self.default_stage tail = None @@ -1065,10 +1076,13 @@ class SConsTimer(object): if self.chdir: os.chdir(self.chdir) - logfile_path = lambda x: os.path.join(self.chdir, x) - if not args: + def _logfile_path_join(x): + return os.path.join(self.chdir, x) + logfile_path = _logfile_path_join + + if not args: pattern = '%s*.log' % self.prefix args = self.args_to_files([pattern], tail) @@ -1372,7 +1386,12 @@ class SConsTimer(object): def do_time(self, argv): format = 'ascii' - logfile_path = lambda x: x + + def _logfile_path(x): + return x + + logfile_path = _logfile_path + tail = None which = 'total' @@ -1422,10 +1441,13 @@ class SConsTimer(object): if self.chdir: os.chdir(self.chdir) - logfile_path = lambda x: os.path.join(self.chdir, x) - if not args: + def _logfile_path_join(x): + return os.path.join(self.chdir, x) + logfile_path = _logfile_path_join + + if not args: pattern = '%s*.log' % self.prefix args = self.args_to_files([pattern], tail) -- cgit v0.12 From ac04f36c2aae0408a5f024dfbdc56a9751e8c144 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sun, 12 Apr 2020 09:49:33 -0600 Subject: [PR #3611] implicit deps fix review comment [ci skip] The updated wording suggested any non-zero value for IMPLICIT_COMMAND_DEPENDENCIES was the same as the default, but in fact values >1 have a different meaning. In updating the wording, nocited that Action.py didn't recognize False-like values the same way as they were listed, or like other parts of scons, where there are multiple false-like values - updated. Signed-off-by: Mats Wichmann --- src/engine/SCons/Action.py | 2 +- src/engine/SCons/Action.xml | 19 +++++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/engine/SCons/Action.py b/src/engine/SCons/Action.py index c02dd33..5bc85d5 100644 --- a/src/engine/SCons/Action.py +++ b/src/engine/SCons/Action.py @@ -964,7 +964,7 @@ class CommandAction(_ActionAction): if is_String(icd) and icd[:1] == '$': icd = env.subst(icd) - if not icd or icd in ('0', 'None'): + if not icd or str(icd).lower in ('0', 'none', 'false', 'no', 'off'): return [] try: diff --git a/src/engine/SCons/Action.xml b/src/engine/SCons/Action.xml index 107e389..b519cdc 100644 --- a/src/engine/SCons/Action.xml +++ b/src/engine/SCons/Action.xml @@ -50,8 +50,10 @@ in the &consenv; used to execute the command. The default is the same as setting the &consvar; &cv-IMPLICIT_COMMAND_DEPENDENCIES; -to a true value (True, -any non-zero number, etc.). +to a True-like value (true, +yes, +or 1 - but not a number +greater than one, as that has a different meaning). @@ -68,10 +70,11 @@ is applied to each segment. If &cv-IMPLICIT_COMMAND_DEPENDENCIES; -is set to a false value -(None, -False, -0, +is set to a False-like value +(none, +false, +no, +0, etc.), then the implicit dependency will not be added to the targets @@ -80,7 +83,7 @@ built with that &consenv;. If &cv-IMPLICIT_COMMAND_DEPENDENCIES; -is set to integer 2 or higher, +is set to 2 or higher, then that number of arguments in the command line will be scanned for relative or absolute paths. If any are present, they will be added as @@ -97,7 +100,7 @@ to the working directory. If &cv-IMPLICIT_COMMAND_DEPENDENCIES; -is set to all, +is set to all, then all arguments in the command line will be scanned for relative or absolute paths. If any are present, they will be added as -- cgit v0.12 From d3f1aa7d7225bff44a18b7ccaad52af071494dab Mon Sep 17 00:00:00 2001 From: William Deegan Date: Tue, 14 Apr 2020 10:39:53 -0700 Subject: [ci skip] adding notice that python bootstrap.py is now obsolete --- bootstrap.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bootstrap.py b/bootstrap.py index 7d8528a..3bf53ea 100755 --- a/bootstrap.py +++ b/bootstrap.py @@ -220,6 +220,10 @@ def main(): if __name__ == "__main__": + print("Please use") + print("python scripts/scons.py") + print("Instead of python bootstrap.py. Bootstrap.py is obsolete") + sys.exit(-1) main() # Local Variables: -- cgit v0.12 From 9ec99deef3afe9d6230b29cc87cfc5ecc9916186 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Thu, 16 Apr 2020 11:16:37 -0600 Subject: Update generation of scons-xml entities [ci skip] PR #3602 introduced a new problem - the documents that include the generated functions and builders files now don't validate. The global function/environment method name was bolded by using , but Docbook doesn't allow that inside a element (it actually works fine with the processing tools we use, but does fail validation). Rework the idea: use and for the markup, and change the way those are rendered in html (man/html.xsl and user/html.xsl) - the PDF already rendered these in bold so no change needed there. Also don't wrap the whole contents of the element in , which would have left the argument list in regular font which the function name and instance name in monospace - an odd look. So the argument list was wrapped in , since that's what they are. Don't bother to try to parse it down into individual args, just do the whole chunk, less the parentheses. Signed-off-by: Mats Wichmann --- bin/scons-proc.py | 51 +++++++++++++++++++++++++++++++++++---------------- doc/man/html.xsl | 8 +++++++- doc/user/html.xsl | 9 +++++++-- 3 files changed, 49 insertions(+), 19 deletions(-) diff --git a/bin/scons-proc.py b/bin/scons-proc.py index ac52894..d6e5030 100644 --- a/bin/scons-proc.py +++ b/bin/scons-proc.py @@ -254,6 +254,7 @@ class Proxy(object): ## return self.__dict__ < other.__dict__ class SConsThing(Proxy): + """Base class for the SConsDoc special elements""" def idfunc(self): return self.name @@ -263,6 +264,7 @@ class SConsThing(Proxy): return [e] class Builder(SConsThing): + """Generate the descriptions and entities for elements""" description = 'builder' prefix = 'b-' tag = 'function' @@ -272,20 +274,20 @@ class Builder(SConsThing): builders don't show a full signature, just func() """ + # build term for global function gterm = stf.newNode("term") - sig = stf.newSubNode(gterm, "literal") - func = stf.newSubNode(sig, "emphasis", role="bold") + func = stf.newSubNode(gterm, "function") stf.setText(func, self.name) stf.setTail(func, '()') + # build term for env. method mterm = stf.newNode("term") - sig = stf.newSubNode(mterm, "literal") - inst = stf.newSubNode(sig, "replaceable") + inst = stf.newSubNode(mterm, "parameter") stf.setText(inst, "env") stf.setTail(inst, ".") - func = stf.newSubNode(sig, "emphasis", role="bold") - stf.setText(func, self.name) - stf.setTail(func, '()') + meth = stf.newSubNode(mterm, "methodname") + stf.setText(meth, self.name) + stf.setTail(meth, '()') return [gterm, mterm] @@ -293,6 +295,7 @@ class Builder(SConsThing): return self.name class Function(SConsThing): + """Generate the descriptions and entities for elements""" description = 'function' prefix = 'f-' tag = 'function' @@ -314,23 +317,37 @@ class Function(SConsThing): signature = 'both' if stf.hasAttribute(arg, 'signature'): signature = stf.getAttribute(arg, 'signature') - s = stf.getText(arg).strip() + sig = stf.getText(arg).strip()[1:-1] # strip (), temporarily if signature in ('both', 'global'): + # build term for global function gterm = stf.newNode("term") - sig = stf.newSubNode(gterm, "literal") - func = stf.newSubNode(sig, "emphasis", role="bold") + func = stf.newSubNode(gterm, "function") stf.setText(func, self.name) - stf.setTail(func, s) + if sig: + # if there are parameters, use that entity + stf.setTail(func, "(") + s = stf.newSubNode(gterm, "parameter") + stf.setText(s, sig) + stf.setTail(s, ")") + else: + stf.setTail(func, "()") tlist.append(gterm) if signature in ('both', 'env'): + # build term for env. method mterm = stf.newNode("term") - sig = stf.newSubNode(mterm, "literal") - inst = stf.newSubNode(sig, "replaceable") + inst = stf.newSubNode(mterm, "replaceable") stf.setText(inst, "env") stf.setTail(inst, ".") - func = stf.newSubNode(sig, "emphasis", role="bold") - stf.setText(func, self.name) - stf.setTail(func, s) + meth = stf.newSubNode(mterm, "methodname") + stf.setText(meth, self.name) + if sig: + # if there are parameters, use that entity + stf.setTail(meth, "(") + s = stf.newSubNode(mterm, "parameter") + stf.setText(s, sig) + stf.setTail(s, ")") + else: + stf.setTail(meth, "()") tlist.append(mterm) if not tlist: @@ -341,6 +358,7 @@ class Function(SConsThing): return self.name class Tool(SConsThing): + """Generate the descriptions and entities for elements""" description = 'tool' prefix = 't-' tag = 'literal' @@ -352,6 +370,7 @@ class Tool(SConsThing): return self.name class Variable(SConsThing): + """Generate the descriptions and entities for elements""" description = 'construction variable' prefix = 'cv-' tag = 'envar' diff --git a/doc/man/html.xsl b/doc/man/html.xsl index 00cc782..d4f33da 100644 --- a/doc/man/html.xsl +++ b/doc/man/html.xsl @@ -28,7 +28,7 @@ xmlns:fo="http://www.w3.org/1999/XSL/Format" version="1.0"> - + @@ -54,6 +54,12 @@ reference title set toc,title + + + + + + diff --git a/doc/user/html.xsl b/doc/user/html.xsl index 0c215cf..2cef43d 100644 --- a/doc/user/html.xsl +++ b/doc/user/html.xsl @@ -28,7 +28,7 @@ xmlns:fo="http://www.w3.org/1999/XSL/Format" version="1.0"> - + @@ -54,9 +54,14 @@ reference toc,title set toc,title + + + + + + - -- cgit v0.12 From 88449724330bb0864b74d6ff1b30b26eb03eff4b Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Thu, 9 Apr 2020 09:45:13 -0600 Subject: Update markup on TARGETS, SOURCES, ARGUMENTS [ci skip] Use to mark up variables in these three areas, also a few other variables that were marked up differently. Sometimes that markup comes in the form of using an existing entity (&BUILD_TARGETS; for example), which is already suitably marked up in the entity definition. Use to mark up when they're in the context of function params. The parse_flags kwarg now includes a link to the env.ParseFlags method, since the latter describes how the args are distributed. Signed-off-by: Mats Wichmann --- doc/man/scons.xml | 106 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 55 insertions(+), 51 deletions(-) diff --git a/doc/man/scons.xml b/doc/man/scons.xml index 5630e93..6c46462 100644 --- a/doc/man/scons.xml +++ b/doc/man/scons.xml @@ -1185,11 +1185,11 @@ and re-initialize the dependency graph from scratch. build[OPTIONS] [TARGETS] ... Builds the specified -TARGETS +TARGETS (and their dependencies) with the specified SCons command-line -OPTIONS. +OPTIONS. b and scons @@ -1232,9 +1232,10 @@ which only happens once at the beginning of interactive mode). clean[OPTIONS] [TARGETS] ... Cleans the specified -TARGETS +TARGETS (and their dependencies) -with the specified options. +with the specified +OPTIONS. c is a synonym. This command is itself a synonym for @@ -2000,15 +2001,15 @@ These warnings are enabled by default. Warnings about attempts to set the reserved &consvar; names -CHANGED_SOURCES, -CHANGED_TARGETS, -TARGET, -TARGETS, -SOURCE, -SOURCES, -UNCHANGED_SOURCES +CHANGED_SOURCES, +CHANGED_TARGETS, +TARGET, +TARGETS, +SOURCE, +SOURCES, +UNCHANGED_SOURCES or -UNCHANGED_TARGETS. +UNCHANGED_TARGETS. These warnings are disabled by default. @@ -2105,7 +2106,7 @@ env['BAR'] = 'bar' As a convenience, &consvars; may also be set or modified by the -parse_flags +parse_flags keyword argument, which applies the &f-link-env-MergeFlags; method (described below) to the argument value @@ -2119,11 +2120,15 @@ env = Environment(parse_flags='-Iinclude -DEBUG -lm') This example adds 'include' to -CPPPATH, +the CPPPATH &consvar; 'EBUG' to -CPPDEFINES, +CPPDEFINES, and 'm' to -LIBS. +LIBS. +&f-link-env-ParseFlags; describes how these arguments +are distributed to &consvars;. + + By default, a new &consenv; is initialized with a set of builder methods @@ -2148,21 +2153,21 @@ and suffixes appropriate for that platform. Note that the win32 platform adds the -SystemDrive +SystemDrive and -SystemRoot +SystemRoot variables from the user's external environment to the &consenv;'s -ENV +ENV dictionary. This is so that any executed commands that use sockets to connect with other systems (such as fetching source files from external CVS repository specifications like -:pserver:anonymous@cvs.sourceforge.net:/cvsroot/scons) +:pserver:anonymous@cvs.sourceforge.net:/cvsroot/scons) will work on Windows systems. -The platform argument may be a function or callable object. +The platform argument may be a function or callable object, in which case the &Environment; method will call it to update the new &consenv;: @@ -2259,15 +2264,15 @@ env = Environment(tools=[my_tool]) may also themselves be two-element lists of the form (toolname, kw_dict). SCons searches for the -toolname +toolname specification file as described above, and passes -kw_dict, +kw_dict, which must be a dictionary, as keyword arguments to the tool's -generate +generate function. The -generate +generate function can use the arguments to modify the tool's behavior by setting up the environment in different ways or otherwise changing its initialization. @@ -2535,7 +2540,7 @@ env.SharedLibrary('word', 'word.cpp', LIBSUFFIXES=['.ocx']) -Note that both the $SHLIBSUFFIX and $LIBSUFFIXES +Note that both the $SHLIBSUFFIX and $LIBSUFFIXES variables must be set if you want SCons to search automatically for dependencies on the non-standard library names; see the descriptions of these variables, below, for more information. @@ -2553,11 +2558,11 @@ env = Program('hello', 'hello.c', parse_flags='-Iinclude -DEBUG -lm') This example adds 'include' to -CPPPATH, +CPPPATH, 'EBUG' to -CPPDEFINES, +CPPDEFINES, and 'm' to -LIBS. +LIBS. Although the builder methods defined by &scons; @@ -2938,7 +2943,7 @@ to affect how you want the build to be performed. - ARGLIST + &ARGLIST; A list of the keyword=value @@ -2971,7 +2976,7 @@ for key, value in ARGLIST: - ARGUMENTS + &ARGUMENTS; A dictionary of all the keyword=value @@ -2981,8 +2986,7 @@ and if a given keyword has more than one value assigned to it on the command line, the last (right-most) value is -the one in the -ARGUMENTS +the one in the &ARGUMENTS; dictionary. Example: @@ -2997,7 +3001,7 @@ else: - BUILD_TARGETS + &BUILD_TARGETS; A list of the targets which &scons; @@ -3043,7 +3047,7 @@ if 'special/program' in BUILD_TARGETS: - COMMAND_LINE_TARGETS + &COMMAND_LINE_TARGETS; A list of the targets explicitly specified on the command line. If there are command line targets, @@ -3066,7 +3070,7 @@ if 'special/program' in COMMAND_LINE_TARGETS: - DEFAULT_TARGETS + &DEFAULT_TARGETS; A list of the target nodes @@ -5610,10 +5614,10 @@ and configured with the same &consenv; into single invocations of the Action object's command line or function. Command lines will typically want to use the -CHANGED_SOURCES +cHANGED_SOURCES &consvar; (and possibly -CHANGED_TARGETS +CHANGED_TARGETS as well) to only pass to the command line those sources that have actually changed since their targets were built. @@ -5944,7 +5948,7 @@ special variables for each command execution: - CHANGED_SOURCES + CHANGED_SOURCES The file names of all sources of the build command that have changed since the target was last built. @@ -5952,7 +5956,7 @@ that have changed since the target was last built. - CHANGED_TARGETS + CHANGED_TARGETS The file names of all targets that would be built from sources that have changed since the target was last built. @@ -5960,7 +5964,7 @@ from sources that have changed since the target was last built. - SOURCE + SOURCE The file name of the source of the build command, or the file name of the first source @@ -5969,14 +5973,14 @@ if multiple sources are being built. - SOURCES + SOURCES The file names of the sources of the build command. - TARGET + TARGET The file name of the target being built, or the file name of the first target @@ -5985,14 +5989,14 @@ if multiple targets are being built. - TARGETS + TARGETS The file names of all targets being built. - UNCHANGED_SOURCES + UNCHANGED_SOURCES The file names of all sources of the build command that have @@ -6245,7 +6249,7 @@ env=Environment(FOO=foo, BAR="$FOO baz") Python function by creating a callable class that stores one or more arguments in an object, and then uses them when the -__call__() +__call__() method is called. Note that in this case, the entire variable expansion must @@ -6347,12 +6351,12 @@ built, not when the SConscript is being read. So if later in the SConscript, the final value will be used. Here's a more interesting example. Note that all of -COND, -FOO, +COND, +FOO, and -BAR are &consvars;, +BAR are &consvars;, and their values are substituted into the final command. -FOO is a list, so its elements are interpolated +FOO is a list, so its elements are interpolated separated by spaces. @@ -6929,7 +6933,7 @@ env.Program(target = 'bar', source = 'bar.c') You also can use other Pythonic techniques to add -to the BUILDERS &consvar;, such as: +to the BUILDERS &consvar;, such as: env = Environment() -- cgit v0.12 From 944270dc1fd82c6fbb43f414f0ae87b2d3fe3976 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Thu, 16 Apr 2020 14:17:45 -0600 Subject: [PR #3612] doc update: cvars are [ci skip] Consider construction variable as environment variables, using that markup in scons.xml and generating it into the variables.gen file (which is included in man and userguide). If possible, use the entity from variables.mod, which already has the envar markup generated. Signed-off-by: Mats Wichmann --- bin/scons-proc.py | 12 ++++++++-- doc/man/scons.xml | 71 ++++++++++++++++++++++++++----------------------------- 2 files changed, 44 insertions(+), 39 deletions(-) diff --git a/bin/scons-proc.py b/bin/scons-proc.py index 4370004..30832a0 100644 --- a/bin/scons-proc.py +++ b/bin/scons-proc.py @@ -273,7 +273,7 @@ class Builder(SConsThing): """ # build term for global function gterm = stf.newNode("term") - func = stf.newSubNode(gterm, "function") + func = stf.newSubNode(gterm, Builder.tag) stf.setText(func, self.name) stf.setTail(func, '()') @@ -282,6 +282,7 @@ class Builder(SConsThing): inst = stf.newSubNode(mterm, "parameter") stf.setText(inst, "env") stf.setTail(inst, ".") + # we could use here, but it's a "method" meth = stf.newSubNode(mterm, "methodname") stf.setText(meth, self.name) stf.setTail(meth, '()') @@ -318,7 +319,7 @@ class Function(SConsThing): if signature in ('both', 'global'): # build term for global function gterm = stf.newNode("term") - func = stf.newSubNode(gterm, "function") + func = stf.newSubNode(gterm, Function.tag) stf.setText(func, self.name) if sig: # if there are parameters, use that entity @@ -335,6 +336,7 @@ class Function(SConsThing): inst = stf.newSubNode(mterm, "replaceable") stf.setText(inst, "env") stf.setTail(inst, ".") + # we could use here, but it's a "method" meth = stf.newSubNode(mterm, "methodname") stf.setText(meth, self.name) if sig: @@ -371,6 +373,12 @@ class Variable(SConsThing): description = 'construction variable' prefix = 'cv-' tag = 'envar' + + def xml_terms(self): + term = stf.newNode("term") + var = stf.newSubNode(term, Variable.tag) + stf.setText(var, self.name) + return [term] def entityfunc(self): return '$' + self.name diff --git a/doc/man/scons.xml b/doc/man/scons.xml index 6c46462..c52a4b8 100644 --- a/doc/man/scons.xml +++ b/doc/man/scons.xml @@ -2001,15 +2001,15 @@ These warnings are enabled by default. Warnings about attempts to set the reserved &consvar; names -CHANGED_SOURCES, -CHANGED_TARGETS, -TARGET, -TARGETS, -SOURCE, -SOURCES, -UNCHANGED_SOURCES +&cv-CHANGED_SOURCES;, +&cv-CHANGED_TARGETS;, +&cv-TARGET;, +&cv-TARGETS;, +&cv-SOURCE;, +&cv-SOURCES;, +&cv-UNCHANGED_SOURCES; or -UNCHANGED_TARGETS. +&cv-UNCHANGED_TARGETS;. These warnings are disabled by default. @@ -2120,11 +2120,11 @@ env = Environment(parse_flags='-Iinclude -DEBUG -lm') This example adds 'include' to -the CPPPATH &consvar; +the CPPPATH &consvar; 'EBUG' to -CPPDEFINES, +CPPDEFINES, and 'm' to -LIBS. +LIBS. &f-link-env-ParseFlags; describes how these arguments are distributed to &consvars;. @@ -2153,12 +2153,12 @@ and suffixes appropriate for that platform. Note that the win32 platform adds the -SystemDrive +SystemDrive and -SystemRoot +SystemRoot variables from the user's external environment to the &consenv;'s -ENV +ENV dictionary. This is so that any executed commands that use sockets to connect with other systems @@ -2540,7 +2540,7 @@ env.SharedLibrary('word', 'word.cpp', LIBSUFFIXES=['.ocx']) -Note that both the $SHLIBSUFFIX and $LIBSUFFIXES +Note that both the $SHLIBSUFFIX and $LIBSUFFIXES variables must be set if you want SCons to search automatically for dependencies on the non-standard library names; see the descriptions of these variables, below, for more information. @@ -2558,11 +2558,11 @@ env = Program('hello', 'hello.c', parse_flags='-Iinclude -DEBUG -lm') This example adds 'include' to -CPPPATH, +CPPPATH, 'EBUG' to -CPPDEFINES, +CPPDEFINES, and 'm' to -LIBS. +LIBS. Although the builder methods defined by &scons; @@ -5614,11 +5614,8 @@ and configured with the same &consenv; into single invocations of the Action object's command line or function. Command lines will typically want to use the -cHANGED_SOURCES -&consvar; -(and possibly -CHANGED_TARGETS -as well) +&cv-CHANGED_SOURCES; &consvar; +(and possibly &cv-CHANGED_TARGETS; as well) to only pass to the command line those sources that have actually changed since their targets were built. @@ -5948,7 +5945,7 @@ special variables for each command execution: - CHANGED_SOURCES + &cv-CHANGED_SOURCES; The file names of all sources of the build command that have changed since the target was last built. @@ -5956,7 +5953,7 @@ that have changed since the target was last built. - CHANGED_TARGETS + &cv-CHANGED_TARGETS; The file names of all targets that would be built from sources that have changed since the target was last built. @@ -5964,7 +5961,7 @@ from sources that have changed since the target was last built. - SOURCE + &cv-SOURCE; The file name of the source of the build command, or the file name of the first source @@ -5973,14 +5970,14 @@ if multiple sources are being built. - SOURCES + &cv-SOURCES; The file names of the sources of the build command. - TARGET + &cv-TARGET; The file name of the target being built, or the file name of the first target @@ -5989,14 +5986,14 @@ if multiple targets are being built. - TARGETS + &cv-TARGETS; The file names of all targets being built. - UNCHANGED_SOURCES + &cv-UNCHANGED_SOURCES; The file names of all sources of the build command that have @@ -6006,7 +6003,7 @@ changed since the target was last built. - UNCHANGED_TARGETS + &cv-UNCHANGED_TARGETS; The file names of all targets that would be built from sources that have @@ -6187,7 +6184,7 @@ ${SOURCE.rsrcdir} => /usr/repository/src to enclose arbitrary Python code to be evaluated. (In fact, this is how the above modifiers are substituted, they are simply attributes of the Python objects -that represent TARGET, SOURCES, etc.) +that represent &cv-TARGET;, &cv-SOURCES;, etc.) See the section "Python Code Substitution" below, for more thorough examples of how this can be used. @@ -6351,12 +6348,12 @@ built, not when the SConscript is being read. So if later in the SConscript, the final value will be used. Here's a more interesting example. Note that all of -COND, -FOO, +COND, +FOO, and -BAR are &consvars;, +BAR are &consvars;, and their values are substituted into the final command. -FOO is a list, so its elements are interpolated +FOO is a list, so its elements are interpolated separated by spaces. @@ -6933,7 +6930,7 @@ env.Program(target = 'bar', source = 'bar.c') You also can use other Pythonic techniques to add -to the BUILDERS &consvar;, such as: +to the BUILDERS &consvar;, such as: env = Environment() -- cgit v0.12 From dd0eb0294ac23ca70ab7b083ff25762ce4ee6ab3 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Tue, 14 Apr 2020 13:43:24 -0600 Subject: Docs: update the Configure Context sections [ci skip] Apply new formatting styles. Tweak a lot of wordings. Make sure manpage consistently describes return types. Signed-off-by: Mats Wichmann --- doc/man/scons.xml | 565 +++++++++++++++++++++++++++-------------------------- doc/user/sconf.xml | 95 +++++---- 2 files changed, 343 insertions(+), 317 deletions(-) diff --git a/doc/man/scons.xml b/doc/man/scons.xml index c52a4b8..f7eed2d 100644 --- a/doc/man/scons.xml +++ b/doc/man/scons.xml @@ -3209,8 +3209,7 @@ env = Environment(CC="cc") or when copying a &consenv; using the -Clone -method: +&f-link-Clone; method: env2 = env.Clone(CC="cl.exe") @@ -3221,67 +3220,75 @@ env2 = env.Clone(CC="cl.exe") Configure Contexts -&scons; +&SCons; supports a -configure context, +&configure_context;, an integrated mechanism similar to the various AC_CHECK macros in GNU &Autoconf; -for testing for the existence of C header -files, libraries, etc. -In contrast to &Autoconf;, +for testing the existence of external items needed +for the build, such as C header files, libraries, etc. +The mechanism is portable across platforms. + + + &scons; -does not maintain an explicit cache of the tested values, +does not maintain an explicit cache of the tested values +(this is different than &Autoconf;), but uses its normal dependency tracking to keep the checked values up to date. However, users may override this behaviour with the command line option. -To create a configure context: - - Configure(env, [custom_tests, conf_dir, log_file, config_h, clean, help]) - env.Configure([custom_tests, conf_dir, log_file, config_h, clean, help]) + Configure(env, [custom_tests, conf_dir, log_file, config_h, clean, help]) + env.Configure([custom_tests, conf_dir, log_file, config_h, clean, help]) -Create a configure context, which tracks information -discovered while running tests. The context includes a -&consenv;, which is used when running the tests and -which is updated with the check results. -When the context is complete, the (possibly modified) -environment is returned). Only one context may be active +Create a &configure_context;, which tracks information +discovered while running tests. The context includes a local &consenv; +(available as context.env) +which is used when running the tests and +which can be updated with the check results. +Only one context may be active at a time (since 4.0, &scons; will raise an exception -if this rule is not followed), but a new context can be created -after the active one is completed. - -The required env argument -specifies the environment for building the tests. -custom_tests -is a dictionary containing custom tests +on an attempt to create a new context when there is +an active context), but a new context can be created +after the active one is completed. +For the global function form, the required env +describes the initial values for the context's local &consenv;; +for the &consenv; method form the instance provides the values. + +custom_tests +specifies a dictionary containing custom tests (see the section on custom tests below). -By default, no custom tests are added to the configure context. -conf_dir +The default value is None, +meaning no custom tests are added to the &configure_context;. + +conf_dir specifies a directory where the test cases are built. -Note that this directory is not used for building -normal targets. -The default value is the directory -#/.sconf_temp. -log_file +This directory is not used for building normal targets. +The default value is +#/.sconf_temp. + +log_file specifies a file which collects the output from commands that are executed to check for the existence of header files, libraries, etc. -The default is the file #/config.log. +The default is #/config.log. If you are using the &VariantDir; function, -you may want to specify a subdirectory under your variant directory. -config_h +you may want to specify a subdirectory under your variant directory. + +config_h specifies a C header file where the results of tests -will be written, e.g. +will be written. The results will consist of lines like #define HAVE_STDIO_H, #define HAVE_LIBM, etc. +Customarily, the name chosen is config.h. The default is to not write a -config.h +config_h file. You can specify the same -config.h +config_h file in multiple calls to &Configure;, in which case &SCons; will concatenate all results in the specified file. @@ -3289,19 +3296,17 @@ Note that &SCons; uses its normal dependency checking to decide if it's necessary to rebuild the specified -config_h +config_h file. This means that the file is not necessarily re-built each time scons is run, but is only rebuilt if its contents will have changed and some target that depends on the -config_h +config_h file is being built. - -The optional -clean +The clean and -help +help arguments can be used to suppress execution of the configuration tests when the / @@ -3309,15 +3314,15 @@ or // options are used, respectively. The default behavior is always to execute -configure context tests, +&configure_context; tests, since the results of the tests may affect the list of targets to be cleaned or the help text. If the configure tests do not affect these, then you may add the -clean=False + or -help=False + arguments (or both) to avoid unnecessary test execution. @@ -3325,19 +3330,19 @@ to avoid unnecessary test execution. - SConf.Finish(context) - context.Finish() + SConf.Finish(context) + context.Finish() This method must be called after configuration is done. Though required, this is not enforced except -if &Configure; is called when there still an active context, +if &Configure; is called again while there is still an active context, in which case an exception is raised. &Finish; returns the environment as modified during the course of running the configuration checks. After this method is called, no further checks can be performed with this configuration context. However, you can create a new -configure context to perform additional checks. +&configure_context; to perform additional checks. @@ -3357,19 +3362,27 @@ if conf.CheckLibWithHeader("qt", "qapp.h", "c++", "QApplication qapp(0,0);"): env = conf.Finish() -A configure context +A &configure_context; has the following predefined methods which -can be used to perform checks: +can be used to perform checks. Where +language is a required or +optional parameter, the choice can currently +be C or C++. The spellings accepted for +C are C or c; +for C++ the value can be +CXX, cxx, C++ +or c++. + - SConf.CheckHeader(context, header, [include_quotes, language]) - context.CheckHeader(header, [include_quotes, language]) + SConf.CheckHeader(context, header, [include_quotes, language]) + context.CheckHeader(header, [include_quotes, language]) Checks if -header +header is usable in the specified language. -header +header may be a list, in which case the last item in the list is the header file to be checked, @@ -3379,32 +3392,32 @@ header files whose lines should precede the header line being checked for. The optional argument -include_quotes +include_quotes must be a two character string, where the first character denotes the opening quote and the second character denotes the closing quote. -By default, both characters are " (double quote). +By default, both characters are " (double quote). The optional argument -language +language should be either C or C++ and selects the compiler to be used for the check. -Returns 1 on success and 0 on failure. +Returns a boolean indicating success or failure. - SConf.CheckCHeader(context, header, [include_quotes]) - context.CheckCHeader(header, [include_quotes]) + SConf.CheckCHeader(context, header, [include_quotes]) + context.CheckCHeader(header, [include_quotes]) This is a wrapper around -SConf.CheckHeader +SConf.CheckHeader which checks if -header +header is usable in the C language. -header +header may be a list, in which case the last item in the list is the header file to be checked, @@ -3414,25 +3427,25 @@ header files whose lines should precede the header line being checked for. The optional argument -include_quotes +include_quotes must be a two character string, where the first character denotes the opening -quote and the second character denotes the closing quote (both default -to \N'34'). -Returns 1 on success and 0 on failure. +quote and the second character denotes the closing quote. +By default, both characters are " (double quote). +Returns a boolean indicating success or failure. - SConf.CheckCXXHeader(context, header, [include_quotes]) - context.CheckCXXHeader(header, [include_quotes]) + SConf.CheckCXXHeader(context, header, [include_quotes]) + context.CheckCXXHeader(header, [include_quotes]) This is a wrapper around -SConf.CheckHeader +SConf.CheckHeader which checks if -header +header is usable in the C++ language. -header +header may be a list, in which case the last item in the list is the header file to be checked, @@ -3442,25 +3455,32 @@ header files whose lines should precede the header line being checked for. The optional argument -include_quotes +include_quotes must be a two character string, where the first character denotes the opening -quote and the second character denotes the closing quote (both default -to \N'34'). -Returns 1 on success and 0 on failure. +quote and the second character denotes the closing quote. +By default, both characters are " (double quote). +Returns a boolean indicating success or failure. - SConf.CheckFunc(context,, function_name, [header, language]) - context.CheckFunc(function_name, [header, language]) + SConf.CheckFunc(context, function_name, [header, language]) + context.CheckFunc(function_name, [header, language]) Checks if the specified -C or C++ function is available. -function_name +C or C++ library function is available based on the +context's local environment settings (that is, using +the values of CFLAGS, +CPPFLAGS, LIBS +or other relevant &consvars;). + + + +function_name is the name of the function to check for. The optional -header +header argument is a string that will be placed at the top @@ -3476,77 +3496,65 @@ extern "C" char function_name(); -The optional -language -argument should be -C -or -C++ -and selects the compiler to be used for the check; -the default is "C". + +Returns an empty string on success, a string containing +an error message on failure. + - SConf.CheckLib(context, [library, symbol, header, language, autoadd=1]) - context.CheckLib([library, symbol, header, language, autoadd=1]) + SConf.CheckLib(context, [library, symbol, header, language, autoadd=True]) + context.CheckLib([library, symbol, header, language, autoadd=True]) Checks if -library +library provides -symbol. -If the value of -autoadd -is 1 and the library provides the specified -symbol, -appends the library to the LIBS &consvar; -library -may also be None (the default), +symbol. +If +autoadd +is true (the default) and the library provides the specified +symbol, +appends the library to the LIBS &consvar; +library +may also be None (the default), in which case -symbol -is checked with the current LIBS variable, +symbol +is checked with the current LIBS variable, or a list of library names, in which case each library in the list will be checked for -symbol. +symbol. If -symbol +symbol is not set or is -None, +None, then -SConf.CheckLib() +SConf.CheckLib just checks if you can link against the specified -library. -The optional -language -argument should be -C -or -C++ -and selects the compiler to be used for the check; -the default is "C". -The default value for -autoadd -is 1. -This method returns 1 on success and 0 on error. +library. +Note though it is legal syntax, it would +not be very useful to call this method +with library +and symbol both +omitted or None. +Returns a boolean indicating success or failure. - SConf.CheckLibWithHeader(context, library, header, language, [call, autoadd]) - context.CheckLibWithHeader(library, header, language, [call, autoadd]) + SConf.CheckLibWithHeader(context, library, header, language, [call, autoadd=True]) + context.CheckLibWithHeader(library, header, language, [call, autoadd=True]) -In contrast to the -SConf.CheckLib -call, this call provides a more sophisticated way to check against libraries. -Again, -library +Provides a more sophisticated way to check against libraries then the +SConf.CheckLib call. +library specifies the library or a list of libraries to check. -header +header specifies a header to check for. -header +header may be a list, in which case the last item in the list is the header file to be checked, @@ -3555,91 +3563,95 @@ header files whose #include lines should precede the header line being checked for. -language -may be one of 'C','c','CXX','cxx','C++' and 'c++'. -call +call can be any valid expression (with a trailing ';'). If -call +call is not set, the default simply checks that you can link against the specified -library. -autoadd -specifies whether to add the library to the environment (only if the check -succeeds). This method returns 1 on success and 0 on error. +library. +autoadd (default true) +specifies whether to add the library to the environment if the check +succeeds. +Returns a boolean indicating success or failure. - SConf.CheckType(context, type_name, [includes, language]) - context.CheckType(type_name, [includes, language]) + SConf.CheckType(context, type_name, [includes, language]) + context.CheckType(type_name, [includes, language]) Checks for the existence of a type defined by -typedef. -type_name +typedef. +type_name specifies the typedef name to check for. -includes +includes is a string containing one or more #include lines that will be inserted into the program that will be run to test for the existence of the type. -The optional -language -argument should be -C -or -C++ -and selects the compiler to be used for the check; -the default is "C". Example: sconf.CheckType('foo_type', '#include "my_types.h"', 'C++') + +Returns an empty string on success, a string containing +an error message on failure. + + - SConf.CheckCC(self) - context.CheckCC(self) + SConf.CheckCC(context) + context.CheckCC() -Checks whether the C compiler (as defined by the CC &consvar;) works -by trying to compile a small source file. +Checks whether the C compiler (as defined by the +CC &consvar;) works +by trying to compile a small source file. +Returns a boolean indicating success or failure. By default, SCons only detects if there is a program with the correct name, not if it is a functioning compiler. -This uses the exact same command than the one used by the object builder for C -source file, so it can be used to detect if a particular compiler flag works or +This uses the exact same command as the one used by the object builder for C +source files, so it can be used to detect if a particular compiler flag works or not. - SConf.CheckCXX(self) - context.CheckCXX(self) + SConf.CheckCXX(context) + context.CheckCXX() -Checks whether the C++ compiler (as defined by the CXX &consvar;) -works by trying to compile a small source file. By default, SCons only detects -if there is a program with the correct name, not if it is a functioning compiler. +Checks whether the C++ compiler (as defined by the +CXX &consvar;) +works by trying to compile a small source file. By default, +SCons only detects if there is a program with the correct name, +not if it is a functioning compiler. +Returns a boolean indicating success or failure. -This uses the exact same command than the one used by the object builder for -CXX source files, so it can be used to detect if a particular compiler flag +This uses the exact same command as the one used by the object builder for +C++ source files, so it can be used to detect if a particular compiler flag works or not. - SConf.CheckSHCC(self) - context.CheckSHCC(self) + SConf.CheckSHCC(context) + context.CheckSHCC() -Checks whether the C compiler (as defined by the SHCC &consvar;) works -by trying to compile a small source file. By default, SCons only detects if -there is a program with the correct name, not if it is a functioning compiler. +Checks whether the shared-object C compiler (as defined by the +SHCC &consvar;) works +by trying to compile a small source file. By default, +SCons only detects if there is a program with the correct name, +not if it is a functioning compiler. +Returns a boolean indicating success or failure. -This uses the exact same command than the one used by the object builder for C +This uses the exact same command as the one used by the object builder for C source file, so it can be used to detect if a particular compiler flag works or not. This does not check whether the object code can be used to build a shared library, only that the compilation (not link) succeeds. @@ -3647,97 +3659,90 @@ library, only that the compilation (not link) succeeds. - SConf.CheckSHCXX(self) - context.CheckSHCXX(self) + SConf.CheckSHCXX(context) + context.CheckSHCXX() -Checks whether the C++ compiler (as defined by the SHCXX &consvar;) -works by trying to compile a small source file. By default, SCons only detects -if there is a program with the correct name, not if it is a functioning compiler. +Checks whether the shared-object C++ compiler (as defined by the +SHCXX &consvar;) +works by trying to compile a small source file. By default, +SCons only detects if there is a program with the correct name, +not if it is a functioning compiler. +Returns a boolean indicating success or failure. -This uses the exact same command than the one used by the object builder for -CXX source files, so it can be used to detect if a particular compiler flag +This uses the exact same command as the one used by the object builder for +C++ source files, so it can be used to detect if a particular compiler flag works or not. This does not check whether the object code can be used to build a shared library, only that the compilation (not link) succeeds. - SConf.CheckTypeSize(context, type_name, [header, language, expect]) - context.CheckTypeSize(type_name, [header, language, expect]) + SConf.CheckTypeSize(context, type_name, [header, language, expect]) + context.CheckTypeSize(type_name, [header, language, expect]) Checks for the size of a type defined by -typedef. -type_name +typedef. +type_name specifies the typedef name to check for. The optional -header +header argument is a string that will be placed at the top of the test file that will be compiled -to check if the function exists; +to check if the type exists; the default is empty. -The optional -language -argument should be -C -or -C++ -and selects the compiler to be used for the check; -the default is "C". -The optional -expect -argument should be an integer. -If this argument is used, -the function will only check whether the type -given in type_name has the expected size (in bytes). +If the optional +expect, +is supplied, it should be an integer size; +&CheckTypeSize; will fail unless +type_name is actually +that size. +Returns the size in bytes, or zero if the type was not found +(or if the size did not match expect). + + For example, CheckTypeSize('short', expect=2) -will return success only if short is two bytes. +will return the size 2 only if short is +actually two bytes. - SConf.CheckDeclaration(context, symbol, [includes, language]) - context.CheckDeclaration(symbol, [includes, language]) + SConf.CheckDeclaration(context, symbol, [includes, language]) + context.CheckDeclaration(symbol, [includes, language]) Checks if the specified -symbol +symbol is declared. -includes +includes is a string containing one or more -#include +#include lines that will be inserted into the program -that will be run to test for the existence of the type. -The optional -language -argument should be -C -or -C++ -and selects the compiler to be used for the check; -the default is "C". +that will be run to test for the existence of the symbol. +Returns a boolean indicating success or failure. - SConf.Define(context, symbol, [value, comment]) - context.Define(symbol, [value, comment]) + SConf.Define(context, symbol, [value, comment]) + context.Define(symbol, [value, comment]) This function does not check for anything, but defines a preprocessor symbol that will be added to the configuration header file. -It is the equivalent of AC_DEFINE, +It is the equivalent of AC_DEFINE, and defines the symbol -name +name with the optional -value +value and the optional comment -comment. +comment. Define Examples: @@ -3787,112 +3792,119 @@ conf.Define("A_SYMBOL", 1, "Set to 1 if you have a symbol") -You can define your own custom checks. +You can define your own custom checks in addition to the predefined checks. -These are passed in a dictionary to the Configure function. +You pass a dictionary of these +to the &Configure; function +as the custom_tests argument. This dictionary maps the names of the checks -to user defined Python callables -(either Python functions or class instances implementing the -__call__ +to the user defined Python callables +(either Python functions or class instances implementing a +__call__ method). -The first argument of the call is always a -CheckContext +Each custom check will be called with a first +argument of a CheckContext, instance followed by the arguments, which must be supplied by the user of the check. -These CheckContext instances define the following methods: +A CheckContext instance defines the following methods: - CheckContext.Message(self, text) + context.Message(text) -Usually called before the check is started. -text -will be displayed to the user, e.g. 'Checking for library X...' +Displays a message, as an indicator of progess. +text +will be displayed, e.g. +Checking for library X.... +Usually called before the check is started. + - CheckContext.Result(self, res) + context.Result(res) -Usually called after the check is done. -res -can be either an integer or a string. In the former case, 'yes' (res != 0) -or 'no' (res == 0) is displayed to the user, in the latter case the -given string is displayed. +Displays a result message, as an indicator of progress. + +res +can be either an integer or a string. If an integer, displays +yes +(if res evaluates True) +or no +(if res evaluates False). +If a string, it is displayed as-is. +Usually called after the check has completed. - CheckContext.TryCompile(self, text, extension) + context.TryCompile(text, extension='') Checks if a file with the specified -extension +extension (e.g. '.c') containing -text +text can be compiled using the environment's -Object -builder. Returns 1 on success and 0 on failure. +&Object; builder. +Returns a boolean indicating success or failure. - CheckContext.TryLink(self, text, extension) + context.TryLink(text, extension='') Checks, if a file with the specified -extension +extension (e.g. '.c') containing -text -can be compiled using the environment's -Program -builder. Returns 1 on success and 0 on failure. +text +can be compiled using the environment's &Program; builder. +Returns a boolean indicating success or failure. - CheckContext.TryRun(self, text, extension) + context.TryRun(text, extension='') -Checks, if a file with the specified -extension +Checks if a file with the specified +extension (e.g. '.c') containing -text +text can be compiled using the environment's -Program -builder. On success, the program is run. If the program +&Program; builder. On success, the program is run. If the program executes successfully (that is, its return status is 0), a tuple (1, outputStr) is returned, where -outputStr +outputStr is the standard output of the program. If the program fails execution (its return status is non-zero), -then (0, '') is returned. +then (0, '') is returned. - CheckContext.TryAction(self, action, [text, extension]) + context.TryAction(action, [text, extension='']) Checks if the specified -action +action with an optional source file (contents -text -, extension -extension -= '' -) can be executed. -action +text, +extension +extension) +can be executed. +action may be anything which can be converted to a &scons; Action. On success, (1, outputStr) is returned, where -outputStr +outputStr is the content of the target file. On failure (0, '') @@ -3901,23 +3913,24 @@ is returned. - CheckContext.TryBuild(self, builder, [text, extension]) + context.TryBuild(builder[, text, extension='']) Low level implementation for testing specific builds; the methods above are based on this method. Given the Builder instance -builder +builder and the optional -text +text of a source file with optional -extension, -this method returns 1 on success and 0 on failure. In addition, -self.lastTarget -is set to the build target node, if the build was successful. +extension, +returns a boolean indicating success or failure. +In addition, +context.lastTarget +is set to the build target node if the build was successful. -Example for implementing and using custom tests: +Example of implementing and using custom tests: def CheckQt(context, qtdir): @@ -3925,7 +3938,7 @@ def CheckQt(context, qtdir): lastLIBS = context.env['LIBS'] lastLIBPATH = context.env['LIBPATH'] lastCPPPATH= context.env['CPPPATH'] - context.env.Append(LIBS = 'qt', LIBPATH = qtdir + '/lib', CPPPATH = qtdir + '/include' ) + context.env.Append(LIBS='qt', LIBPATH=qtdir + '/lib', CPPPATH=qtdir + '/include') ret = context.TryLink(""" #include <qapp.h> int main(int argc, char **argv) { @@ -3934,12 +3947,12 @@ int main(int argc, char **argv) { } """) if not ret: - context.env.Replace(LIBS = lastLIBS, LIBPATH=lastLIBPATH, CPPPATH=lastCPPPATH) + context.env.Replace(LIBS=lastLIBS, LIBPATH=lastLIBPATH, CPPPATH=lastCPPPATH) context.Result( ret ) return ret env = Environment() -conf = Configure( env, custom_tests = { 'CheckQt' : CheckQt } ) +conf = Configure(env, custom_tests = {'CheckQt': CheckQt}) if not conf.CheckQt('/usr/lib/qt'): print('We really need qt!') Exit(1) diff --git a/doc/user/sconf.xml b/doc/user/sconf.xml index 109e468..8961606 100644 --- a/doc/user/sconf.xml +++ b/doc/user/sconf.xml @@ -46,35 +46,29 @@ - &SCons; has integrated support for multi-platform build configuration - similar to that offered by GNU &Autoconf;, - such as - figuring out what libraries or header files - are available on the local system. + &SCons; has integrated support for build configuration + similar in style to GNU &Autoconf;, but designed to be + transparently multi-platform. The configuration system + can help figure out if external build requirements such + as system libraries or header files + are available on the build system. This section describes how to use this &SCons; feature. + (See also the &SCons; man page for additional information). - - - This chapter is still under development, - so not everything is explained as well as it should be. - See the &SCons; man page for additional information. - - -
&Configure_Contexts; The basic framework for multi-platform build configuration - in &SCons; is to attach a &configure_context; to a - construction environment by calling the &Configure; function, - perform a number of checks for + in &SCons; is to create a &configure_context; inside a + &consenv; by calling the &Configure; function, + perform the desired checks for libraries, functions, header files, etc., - and to then call the configure context's &Finish; method + and then call the configure context's &Finish; method to finish off the configuration: @@ -88,7 +82,15 @@ env = conf.Finish() - &SCons; provides a number of basic checks, + The &Finish; call is required; if a new context is + created while a context is active, even in a different + &consenv;, &scons; will complain and exit. + + + + + + &SCons; provides a number of pre-defined basic checks, as well as a mechanism for adding your own custom checks. @@ -125,8 +127,11 @@ env = conf.Finish() Testing the existence of a header file requires knowing what language the header file is. - A configure context has a &CheckCHeader; method - that checks for the existence of a C header file: + This information is supplied in the language + keyword parameter to the &CheckHeader; method. + Since &scons; grew up in a world of C/C++ code, + a &configure_context; also has a &CheckCHeader; method + that specifically checks for the existence of a C header file: @@ -134,7 +139,7 @@ env = conf.Finish() env = Environment() conf = Configure(env) if not conf.CheckCHeader('math.h'): - print 'Math.h must be installed!' + print('Math.h must be installed!') Exit(1) if conf.CheckCHeader('foo.h'): conf.env.Append('-DHAS_FOO_H') @@ -143,10 +148,18 @@ env = conf.Finish() - Note that you can choose to terminate + As shown in the example, depending on the circumstances + you can choose to terminate the build if a given header file doesn't exist, or you can modify the construction environment - based on the existence of a header file. + based on the presence or absence of a header file + (the same applies to any other check). If there are a + many elements to check for, it may be friendlier for + the user if you do not terminate on the first failure, + but track the problems found until the end and report on + all of them, that way the user does not have to iterate + multiple times, each time finding one new dependency that + needs to be installed. @@ -162,7 +175,7 @@ env = conf.Finish() env = Environment() conf = Configure(env) if not conf.CheckCXXHeader('vector.h'): - print 'vector.h must be installed!' + print('vector.h must be installed!') Exit(1) env = conf.Finish() @@ -183,8 +196,8 @@ env = conf.Finish() env = Environment() conf = Configure(env) if not conf.CheckFunc('strcpy'): - print 'Did not find strcpy(), using local version' - conf.env.Append(CPPDEFINES = '-Dstrcpy=my_local_strcpy') + print('Did not find strcpy(), using local version') + conf.env.Append(CPPDEFINES='-Dstrcpy=my_local_strcpy') env = conf.Finish() @@ -197,7 +210,7 @@ env = conf.Finish() Check for the availability of a library using the &CheckLib; method. - You only specify the basename of the library, + You only specify the base part of the library name, you don't need to add a lib prefix or a .a or .lib suffix: @@ -207,7 +220,7 @@ env = conf.Finish() env = Environment() conf = Configure(env) if not conf.CheckLib('m'): - print 'Did not find libm.a or m.lib, exiting!' + print('Did not find libm.a or m.lib, exiting!') Exit(1) env = conf.Finish() @@ -227,8 +240,8 @@ env = conf.Finish() env = Environment() conf = Configure(env) -if not conf.CheckLibWithHeader('m', 'math.h', 'c'): - print 'Did not find libm.a or m.lib, exiting!' +if not conf.CheckLibWithHeader('m', 'math.h', language='c'): + print('Did not find libm.a or m.lib, exiting!') Exit(1) env = conf.Finish() @@ -257,8 +270,8 @@ env = conf.Finish() env = Environment() conf = Configure(env) if not conf.CheckType('off_t'): - print 'Did not find off_t typedef, assuming int' - conf.env.Append(CCFLAGS = '-Doff_t=int') + print('Did not find off_t typedef, assuming int') + conf.env.Append(CCFLAGS='-Doff_t=int') env = conf.Finish() @@ -276,8 +289,8 @@ env = conf.Finish() env = Environment() conf = Configure(env) if not conf.CheckType('off_t', '#include <sys/types.h>\n'): - print 'Did not find off_t typedef, assuming int' - conf.env.Append(CCFLAGS = '-Doff_t=int') + print('Did not find off_t typedef, assuming int') + conf.env.Append(CCFLAGS='-Doff_t=int') env = conf.Finish() @@ -292,7 +305,7 @@ env = conf.Finish() env = Environment() conf = Configure(env) int_size = conf.CheckTypeSize('unsigned int') -print 'sizeof unsigned int is', int_size +print('sizeof unsigned int is', int_size) env = conf.Finish() @@ -317,7 +330,7 @@ scons: `.' is up to date. env = Environment() conf = Configure(env) if not conf.CheckProg('foobar'): - print 'Unable to find the program foobar on the system' + print('Unable to find the program foobar on the system') Exit(1) env = conf.Finish() @@ -399,7 +412,7 @@ def CheckMyLibrary(context): env = Environment() -conf = Configure(env, custom_tests = {'CheckMyLibrary' : CheckMyLibrary}) +conf = Configure(env, custom_tests={'CheckMyLibrary': CheckMyLibrary}) @@ -436,9 +449,9 @@ def CheckMyLibrary(context): return result env = Environment() -conf = Configure(env, custom_tests = {'CheckMyLibrary' : CheckMyLibrary}) +conf = Configure(env, custom_tests={'CheckMyLibrary': CheckMyLibrary}) if not conf.CheckMyLibrary(): - print 'MyLibrary is not installed!' + print('MyLibrary is not installed!') Exit(1) env = conf.Finish() @@ -516,9 +529,9 @@ Removed foo env = Environment() if not env.GetOption('clean'): - conf = Configure(env, custom_tests = {'CheckMyLibrary' : CheckMyLibrary}) + conf = Configure(env, custom_tests={'CheckMyLibrary': CheckMyLibrary}) if not conf.CheckMyLibrary(): - print 'MyLibrary is not installed!' + print('MyLibrary is not installed!') Exit(1) env = conf.Finish() -- cgit v0.12