From 926bafc7fbe46e9414366e93a1dbca318718af7e Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Mon, 27 Jul 2020 10:06:11 -0600 Subject: Align SetOption doc with settable options list. [skip appveyor] List is recast into a table for readability. Comments in both places remind to keep lists in sync. Signed-off-by: Mats Wichmann --- SCons/Script/Main.xml | 227 ++++++++++++++++++++++--------------------- SCons/Script/SConsOptions.py | 2 +- 2 files changed, 118 insertions(+), 111 deletions(-) diff --git a/SCons/Script/Main.xml b/SCons/Script/Main.xml index 0a8f9dc..1519171 100644 --- a/SCons/Script/Main.xml +++ b/SCons/Script/Main.xml @@ -80,10 +80,10 @@ it will recognize , and so forth as long as there is no other option which could also match to the same abbreviation. Options added via -AddOption do not support +&f-AddOption; do not support the automatic recognition of abbreviations. Instead, to allow specific abbreviations, -include them in the &f-AddOption; call. +include them as synonyms in the &f-AddOption; call itself. @@ -99,7 +99,7 @@ options added with &f-AddOption;. The value may also be set using &f-SetOption; or -env.SetOption(), +&f-env.SetOption;, if conditions in a &SConscript; require overriding any default value. @@ -765,123 +765,130 @@ Multiple targets can be passed in to a single call to -This function provides a way to set a select subset of the scons command -line options from a SConscript file. The options supported are: - - - - - -clean - - -which corresponds to , -and ; - - - - -duplicate - - -which corresponds to ; - - - - -help - - -which corresponds to and ; - - - - -implicit_cache - - -which corresponds to ; - - - - -max_drift - - -which corresponds to ; - - - - -no_exec - - -which corresponds to , , -, -and ; - - - - -num_jobs - - -which corresponds to and ; - - - - -random - - -which corresponds to ; and - - - - -silent - - -which corresponds to . - - - - -no_progress - - -which corresponds to -Q. - - - Note: The initial progress output will still be output as this is done before the SConstruct/SConscript which contains the SetOption is processed - scons: Reading SConscript files ... - -Available since &scons; 4.0. - - - -stack_size - - -which corresponds to --stack-size. - - - - +Sets &scons; option variable name +to value. +These options are all also settable via +&scons; command-line options but the names may differ. +A value set via command-line option will take +precedence over one set with &f-SetOption;, which +allows setting a project default in the scripts and +temporarily overriding it via command line. + + + +The settable variables with their associated command-line options are: + + + + + + + +VariableCommand-line options + + + +clean + +, , + + +diskcheck + + + + +duplicate + + + + +help + +, + + +implicit_cache + + + + + +max_drift + + + + +md5_chunksize + + + + +no_exec + +, , +, , + + + +no_progress + + + + +num_jobs + +, + + +random + + + + +silent + +. + + +stack_size + + + + +warn + +. + + + + + + +See the documentation in the manpage for the +corresponding command line option for information about each specific option. +Option values which are boolean in nature (that is, they are +either on or off) should be set to a true value (True, +1) or a false value (False, +0). + -See the documentation for the -corresponding command line option for information about each specific -option. +If no_progress is set via &f-SetOption; +there will still be initial progress output as &SCons; has +to start reading SConscript files before it can see the +&f-SetOption; in an SConscript file: +scons: Reading SConscript files ... + Example: -SetOption('max_drift', 1) +SetOption('max_drift', True) diff --git a/SCons/Script/SConsOptions.py b/SCons/Script/SConsOptions.py index 9fee74e..e407306 100644 --- a/SCons/Script/SConsOptions.py +++ b/SCons/Script/SConsOptions.py @@ -126,7 +126,7 @@ class SConsValues(optparse.Values): # is not available. raise AttributeError(attr) - + # keep this list in sync with SetOption doc in SCons/Script/Main.xml settable = [ 'clean', 'diskcheck', -- cgit v0.12 From fb5fdb341325c89878dfa5cc2bbdbe96a360154f Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Wed, 29 Jul 2020 08:56:52 -0600 Subject: [PR #3758] be more explicit about syncing SetOption list [ci skip] Signed-off-by: Mats Wichmann --- SCons/Script/Main.xml | 3 ++- SCons/Script/SConsOptions.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/SCons/Script/Main.xml b/SCons/Script/Main.xml index 1519171..ed9f6c3 100644 --- a/SCons/Script/Main.xml +++ b/SCons/Script/Main.xml @@ -779,7 +779,8 @@ temporarily overriding it via command line. The settable variables with their associated command-line options are: - + + diff --git a/SCons/Script/SConsOptions.py b/SCons/Script/SConsOptions.py index e407306..cf203e1 100644 --- a/SCons/Script/SConsOptions.py +++ b/SCons/Script/SConsOptions.py @@ -126,7 +126,8 @@ class SConsValues(optparse.Values): # is not available. raise AttributeError(attr) - # keep this list in sync with SetOption doc in SCons/Script/Main.xml + # keep this list in sync with the SetOption doc in SCons/Script/Main.xml + # search for UPDATE_SETOPTION_DOCS there. settable = [ 'clean', 'diskcheck', -- cgit v0.12 From df8780c8006456e7cd791317c8e92c58d66d7578 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Thu, 30 Jul 2020 08:03:13 -0600 Subject: [PR #3758] expand on SetOption limitations [skip appvyor] Signed-off-by: Mats Wichmann --- SCons/Script/Main.xml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/SCons/Script/Main.xml b/SCons/Script/Main.xml index ed9f6c3..8c707d0 100644 --- a/SCons/Script/Main.xml +++ b/SCons/Script/Main.xml @@ -768,11 +768,15 @@ Multiple targets can be passed in to a single call to Sets &scons; option variable name to value. These options are all also settable via -&scons; command-line options but the names may differ. +&scons; command-line options but the variable name +may differ from the command-line option name (see table). A value set via command-line option will take precedence over one set with &f-SetOption;, which allows setting a project default in the scripts and temporarily overriding it via command line. +Options which affect the reading and processing of SConscript files +are not settable this way, since those files must +be read in order to find the &f-SetOption; call. -- cgit v0.12 From 3d5f678da21d1c0bad501b481e67d0a9ba33542a Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sun, 2 Aug 2020 09:46:19 -0600 Subject: Minor: add a comment about options to manpage source [ci skip] Explain why we don't use this form of listing options in manpage: -A--a-long-opt... i.e. a term for each variant of an option. This is a comment only, no change to actual manpage. Signed-off-by: Mats Wichmann --- doc/man/scons.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/man/scons.xml b/doc/man/scons.xml index b33c035..29ace2d 100644 --- a/doc/man/scons.xml +++ b/doc/man/scons.xml @@ -487,6 +487,11 @@ supports the same command-line options as GNU &Make; and many of those supported by cons. + + + + + -- cgit v0.12 From 44e55878823357414dee54bdbf714713ff22aab8 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sun, 2 Aug 2020 11:46:08 -0600 Subject: tweak varlist options commend in manpage src [ci skip] Signed-off-by: Mats Wichmann --- doc/man/scons.xml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/man/scons.xml b/doc/man/scons.xml index 29ace2d..cfbb772 100644 --- a/doc/man/scons.xml +++ b/doc/man/scons.xml @@ -488,9 +488,10 @@ and many of those supported by cons. - - - + + + + -- cgit v0.12 From bf7eb144df61fd040bd149d06dab9fb4d85208d1 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Wed, 5 Aug 2020 20:20:46 -0700 Subject: Fix zip tool to respect ZIPCOMSTR if it's set after an Environment or Tool('zip') is initialized. (It works fine if set in Environment()) Previous PR was #3659 by Rocco Matano --- CHANGES.txt | 8 ++++---- SCons/Tool/zip.py | 32 ++++++++++++++++++-------------- test/ZIP/ZIPCOMSTR.py | 9 +++++++++ 3 files changed, 31 insertions(+), 18 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index bbd7ae1..08f0aa3 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -8,10 +8,10 @@ NOTE: The 4.0.0 Release of SCons dropped Python 2.7 Support RELEASE VERSION/DATE TO BE FILLED IN LATER - From John Doe: - - - Whatever John Doe did. - + From Rocco Matano: + - Fix Zip tool to respect ZIPCOMSTR. Previously all zip builder calls would yield something + like zip(["test.zip"], ["zip_scons.py"]) and ignore ZIPCOMSTR if ZIPCOM and ZIPCOMSTR + weren't set after the Environment/Tool is initialized. (Explained in PR #3659) RELEASE 4.0.1 - Mon, 16 Jul 2020 16:06:40 -0700 diff --git a/SCons/Tool/zip.py b/SCons/Tool/zip.py index 23d540f..cbf2c16 100644 --- a/SCons/Tool/zip.py +++ b/SCons/Tool/zip.py @@ -42,7 +42,9 @@ import SCons.Util import zipfile -zipcompression = zipfile.ZIP_DEFLATED +zip_compression = zipfile.ZIP_DEFLATED + + def zip(target, source, env): compression = env.get('ZIPCOMPRESSION', 0) zf = zipfile.ZipFile(str(target[0]), 'w', compression) @@ -52,19 +54,20 @@ def zip(target, source, env): for fname in filenames: path = os.path.join(dirpath, fname) if os.path.isfile(path): - zf.write(path, os.path.relpath(path, str(env.get('ZIPROOT', '')))) else: zf.write(str(s), os.path.relpath(str(s), str(env.get('ZIPROOT', '')))) zf.close() -zipAction = SCons.Action.Action(zip, varlist=['ZIPCOMPRESSION']) +# Fix PR #3569 - If you don't specify ZIPCOM and ZIPCOMSTR when creating +# env, then it will ignore ZIPCOMSTR set afterwards. +zipAction = SCons.Action.Action(zip, "$ZIPCOMSTR", varlist=['ZIPCOMPRESSION']) -ZipBuilder = SCons.Builder.Builder(action = SCons.Action.Action('$ZIPCOM', '$ZIPCOMSTR'), - source_factory = SCons.Node.FS.Entry, - source_scanner = SCons.Defaults.DirScanner, - suffix = '$ZIPSUFFIX', - multi = 1) +ZipBuilder = SCons.Builder.Builder(action=SCons.Action.Action('$ZIPCOM', '$ZIPCOMSTR'), + source_factory=SCons.Node.FS.Entry, + source_scanner=SCons.Defaults.DirScanner, + suffix='$ZIPSUFFIX', + multi=1) def generate(env): @@ -75,12 +78,13 @@ def generate(env): bld = ZipBuilder env['BUILDERS']['Zip'] = bld - env['ZIP'] = 'zip' - env['ZIPFLAGS'] = SCons.Util.CLVar('') - env['ZIPCOM'] = zipAction - env['ZIPCOMPRESSION'] = zipcompression - env['ZIPSUFFIX'] = '.zip' - env['ZIPROOT'] = SCons.Util.CLVar('') + env['ZIP'] = 'zip' + env['ZIPFLAGS'] = SCons.Util.CLVar('') + env['ZIPCOM'] = zipAction + env['ZIPCOMPRESSION'] = zip_compression + env['ZIPSUFFIX'] = '.zip' + env['ZIPROOT'] = SCons.Util.CLVar('') + def exists(env): return True diff --git a/test/ZIP/ZIPCOMSTR.py b/test/ZIP/ZIPCOMSTR.py index af9ba57..43566fb 100644 --- a/test/ZIP/ZIPCOMSTR.py +++ b/test/ZIP/ZIPCOMSTR.py @@ -42,12 +42,21 @@ env = Environment(tools=['zip'], ZIPCOM = r'%(_python_)s mycompile.py zip $TARGET $SOURCES', ZIPCOMSTR = 'Zipping $TARGET from $SOURCE') env.Zip('aaa.zip', 'aaa.in') + +# Issue explained in PR #3569 - setting ZIPCOM/ZIPCOMSTR after env initialization +# is ignored and yields zip() instead of desired ZIPCOMSTR> +env2 = Environment(tools=['zip']) +env2['ZIPCOM'] = r'%(_python_)s mycompile.py zip $TARGET $SOURCES' +env2['ZIPCOMSTR']="TESTING ONE TWO THREE $TARGET from $SOURCE" +env2.Zip('aaa2.zip', 'aaa.in') + """ % locals()) test.write('aaa.in', 'aaa.in\n/*zip*/\n') test.run(stdout = test.wrap_stdout("""\ Zipping aaa.zip from aaa.in +TESTING ONE TWO THREE aaa2.zip from aaa.in """)) test.must_match('aaa.zip', "aaa.in\n") -- cgit v0.12 From 2c734a2331df7ae96ce851aa39a7606d0e43469d Mon Sep 17 00:00:00 2001 From: William Deegan Date: Wed, 5 Aug 2020 21:50:12 -0700 Subject: Fix YACC in env not respected if set before Environment/Tool is initalized --- CHANGES.txt | 4 ++-- SCons/Tool/yacc.py | 25 +++++++++++++++---------- test/YACC/YACC-fixture/SConstruct_YACC_before | 4 ++++ test/YACC/YACC.py | 4 ++-- 4 files changed, 23 insertions(+), 14 deletions(-) create mode 100644 test/YACC/YACC-fixture/SConstruct_YACC_before diff --git a/CHANGES.txt b/CHANGES.txt index 09c3400..692872c 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -8,9 +8,9 @@ NOTE: The 4.0.0 Release of SCons dropped Python 2.7 Support RELEASE VERSION/DATE TO BE FILLED IN LATER - From John Doe: + From William Deegan: + - Fix yaccc tool, not respecting YACC set at time of tool initialization. - - Whatever John Doe did. From Adam Gross: - Fix minor bug affecting SCons.Node.FS.File.get_csig()'s usage of the MD5 chunksize. diff --git a/SCons/Tool/yacc.py b/SCons/Tool/yacc.py index 0b62305..c7bdf4c 100644 --- a/SCons/Tool/yacc.py +++ b/SCons/Tool/yacc.py @@ -50,14 +50,14 @@ if sys.platform == 'win32': else: BINS = ["bison", "yacc"] + def _yaccEmitter(target, source, env, ysuf, hsuf): yaccflags = env.subst("$YACCFLAGS", target=target, source=source) flags = SCons.Util.CLVar(yaccflags) targetBase, targetExt = os.path.splitext(SCons.Util.to_String(target[0])) - if '.ym' in ysuf: # If using Objective-C - target = [targetBase + ".m"] # the extension is ".m". - + if '.ym' in ysuf: # If using Objective-C + target = [targetBase + ".m"] # the extension is ".m". # If -d is specified on the command line, yacc will emit a .h # or .hpp file with the same name as the .c or .cpp output file. @@ -75,10 +75,8 @@ def _yaccEmitter(target, source, env, ysuf, hsuf): # be noted and also be cleaned # Bug #2558 if "-v" in flags: - env.SideEffect(targetBase+'.output',target[0]) - env.Clean(target[0],targetBase+'.output') - - + env.SideEffect(targetBase + '.output', target[0]) + env.Clean(target[0], targetBase + '.output') # With --defines and --graph, the name of the file is totally defined # in the options. @@ -94,15 +92,19 @@ def _yaccEmitter(target, source, env, ysuf, hsuf): return (target, source) + def yEmitter(target, source, env): return _yaccEmitter(target, source, env, ['.y', '.yacc'], '$YACCHFILESUFFIX') + def ymEmitter(target, source, env): return _yaccEmitter(target, source, env, ['.ym'], '$YACCHFILESUFFIX') + def yyEmitter(target, source, env): return _yaccEmitter(target, source, env, ['.yy'], '$YACCHXXFILESUFFIX') + def get_yacc_path(env, append_paths=False): """ Find the path to the yacc tool, searching several possible names @@ -118,7 +120,7 @@ def get_yacc_path(env, append_paths=False): bin_path = SCons.Tool.find_program_path( env, prog, - default_paths=CHOCO_DEFAULT_PATH + MINGW_DEFAULT_PATHS + CYGWIN_DEFAULT_PATHS ) + default_paths=CHOCO_DEFAULT_PATH + MINGW_DEFAULT_PATHS + CYGWIN_DEFAULT_PATHS) if bin_path: if append_paths: env.AppendENVPath('PATH', os.path.dirname(bin_path)) @@ -149,13 +151,16 @@ def generate(env): # ignore the return, all we need is for the path to be added _ = get_yacc_path(env, append_paths=True) - env["YACC"] = env.Detect(BINS) + if 'YACC' not in env: + env["YACC"] = env.Detect(BINS) + env['YACCFLAGS'] = SCons.Util.CLVar('') - env['YACCCOM'] = '$YACC $YACCFLAGS -o $TARGET $SOURCES' + env['YACCCOM'] = '$YACC $YACCFLAGS -o $TARGET $SOURCES' env['YACCHFILESUFFIX'] = '.h' env['YACCHXXFILESUFFIX'] = '.hpp' env['YACCVCGFILESUFFIX'] = '.vcg' + def exists(env): if sys.platform == 'win32': return get_yacc_path(env) diff --git a/test/YACC/YACC-fixture/SConstruct_YACC_before b/test/YACC/YACC-fixture/SConstruct_YACC_before new file mode 100644 index 0000000..94f7adf --- /dev/null +++ b/test/YACC/YACC-fixture/SConstruct_YACC_before @@ -0,0 +1,4 @@ +env=Environment(tools=[]) +env2=env.Clone(YACC="SOMETHING_DUMB") +env2.Tool('yacc') +env2.CFile('aaa.y') diff --git a/test/YACC/YACC.py b/test/YACC/YACC.py index b27c2a7..d4cb40e 100644 --- a/test/YACC/YACC.py +++ b/test/YACC/YACC.py @@ -58,8 +58,8 @@ test.must_match('bbb.c', "bbb.yacc" + os.linesep + "myyacc.py" + os.lines test.must_match('ccc.cc', "ccc.yacc" + os.linesep + "myyacc.py" + os.linesep) test.must_match('ddd.m', "ddd.yacc" + os.linesep + "myyacc.py" + os.linesep) - - +test.run(arguments="-n -f SConstruct_YACC_before") +test.fail_test('SOMETHING_DUMB' not in test.stdout(), "YACC is not overridden to be SOMETHING_DUMB") test.pass_test() -- cgit v0.12 From 24a5c34667b509c607f7b0bfd10a5c15f9ce1b0d Mon Sep 17 00:00:00 2001 From: William Deegan Date: Wed, 5 Aug 2020 22:14:16 -0700 Subject: fix typo --- CHANGES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index 692872c..079a760 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -9,7 +9,7 @@ NOTE: The 4.0.0 Release of SCons dropped Python 2.7 Support RELEASE VERSION/DATE TO BE FILLED IN LATER From William Deegan: - - Fix yaccc tool, not respecting YACC set at time of tool initialization. + - Fix yacc tool, not respecting YACC set at time of tool initialization. From Adam Gross: -- cgit v0.12 From 07c091119fc56bc455133060bfbceda36debfb95 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Fri, 7 Aug 2020 06:09:09 -0400 Subject: Rework vc.py caching of installed vcs and remove cached_get_installed_vcs function. Replace external references to cached_get_installed_vcs with get_installed_vcs. Update CHANGES.txt. --- CHANGES.txt | 4 ++++ SCons/Tool/MSCommon/vc.py | 20 +++++++++----------- test/MSVC/MSVC_UWP_APP.py | 2 +- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 7250a9b..cfc3a02 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -8,6 +8,10 @@ NOTE: The 4.0.0 Release of SCons dropped Python 2.7 Support RELEASE VERSION/DATE TO BE FILLED IN LATER + From Joseph Brill: + - MSVC and test updates: Rework the msvc installed versions cache so that it is not + exposed externally and update external references accordingly. + From William Deegan: - Fix yacc tool, not respecting YACC set at time of tool initialization. diff --git a/SCons/Tool/MSCommon/vc.py b/SCons/Tool/MSCommon/vc.py index c6417e9..886288a 100644 --- a/SCons/Tool/MSCommon/vc.py +++ b/SCons/Tool/MSCommon/vc.py @@ -657,16 +657,12 @@ def _check_cl_exists_in_vc_dir(env, vc_dir, msvc_version): return False -def cached_get_installed_vcs(env=None): +def get_installed_vcs(env=None): global __INSTALLED_VCS_RUN - if __INSTALLED_VCS_RUN is None: - ret = get_installed_vcs(env) - __INSTALLED_VCS_RUN = ret - - return __INSTALLED_VCS_RUN + if __INSTALLED_VCS_RUN is not None: + return __INSTALLED_VCS_RUN -def get_installed_vcs(env=None): installed_versions = [] for ver in _VCVER: @@ -687,7 +683,9 @@ def get_installed_vcs(env=None): raise except VisualCException as e: debug('did not find VC %s: caught exception %s' % (ver, str(e))) - return installed_versions + + __INSTALLED_VCS_RUN = installed_versions + return __INSTALLED_VCS_RUN def reset_installed_vcs(): """Make it try again to find VC. This is just for the tests.""" @@ -758,7 +756,7 @@ def get_default_version(env): return msvs_version if not msvc_version: - installed_vcs = cached_get_installed_vcs(env) + installed_vcs = get_installed_vcs(env) debug('installed_vcs:%s' % installed_vcs) if not installed_vcs: #msg = 'No installed VCs' @@ -853,7 +851,7 @@ def msvc_find_valid_batch_script(env, version): warn_msg = "VC version %s not installed. " + \ "C/C++ compilers are most likely not set correctly.\n" + \ " Installed versions are: %s" - warn_msg = warn_msg % (version, cached_get_installed_vcs(env)) + warn_msg = warn_msg % (version, get_installed_vcs(env)) SCons.Warnings.warn(SCons.Warnings.VisualCMissingWarning, warn_msg) continue @@ -948,7 +946,7 @@ def msvc_setup_env(env): SCons.Warnings.warn(SCons.Warnings.VisualCMissingWarning, warn_msg) def msvc_exists(env=None, version=None): - vcs = cached_get_installed_vcs(env) + vcs = get_installed_vcs(env) if version is None: return len(vcs) > 0 return version in vcs diff --git a/test/MSVC/MSVC_UWP_APP.py b/test/MSVC/MSVC_UWP_APP.py index 0f33dda..4bb2f5c 100644 --- a/test/MSVC/MSVC_UWP_APP.py +++ b/test/MSVC/MSVC_UWP_APP.py @@ -82,7 +82,7 @@ test = TestSCons.TestSCons() test.skip_if_not_msvc() -installed_msvc_versions = msvc.cached_get_installed_vcs() +installed_msvc_versions = msvc.get_installed_vcs() # MSVC guaranteed to be at least one version on the system or else # skip_if_not_msvc() function would have skipped the test -- cgit v0.12 From 6e7f1530dedce1445e4d5d9640abaf293ac8032d Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Fri, 7 Aug 2020 07:09:26 -0400 Subject: Remove hard-coded MSCommon from debug logging record filename field. Add method to retrieve truncated relative module file name path. Add filter and derived path field for debug log record processing to produce the correct relative file name information for calls to debug from above MSCommon (e.g, msvsTests). --- CHANGES.txt | 4 ++++ SCons/Tool/MSCommon/common.py | 31 +++++++++++++++++++++++++++++-- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 7250a9b..eae76c8 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -8,6 +8,10 @@ NOTE: The 4.0.0 Release of SCons dropped Python 2.7 Support RELEASE VERSION/DATE TO BE FILLED IN LATER + From Joseph Brill: + - Modify the MSCommon internal-use only debug logging records to contain the correct relative + file path when the debug function is called from outside the MSCommon module. + From William Deegan: - Fix yacc tool, not respecting YACC set at time of tool initialization. diff --git a/SCons/Tool/MSCommon/common.py b/SCons/Tool/MSCommon/common.py index e1a82f2..81004df 100644 --- a/SCons/Tool/MSCommon/common.py +++ b/SCons/Tool/MSCommon/common.py @@ -42,19 +42,46 @@ if LOGFILE == '-': print(message) elif LOGFILE: import logging + modulelist = ( + # root module and parent/root module + 'MSCommon', 'Tool', + # python library and below: correct iff scons does not have a lib folder + 'lib', + # scons modules + 'SCons', 'test', 'scons' + ) + def get_relative_filename(filename, module_list): + if not filename: + return filename + for module in module_list: + try: + ind = filename.rindex(module) + return filename[ind:] + except ValueError: + pass + return filename + class _Debug_Filter(logging.Filter): + # custom filter for module relative filename + def filter(self, record): + relfilename = get_relative_filename(record.pathname, modulelist) + relfilename = relfilename.replace('\\', '/') + record.relfilename = relfilename + return True logging.basicConfig( # This looks like: # 00109ms:MSCommon/vc.py:find_vc_pdir#447: format=( '%(relativeCreated)05dms' - ':MSCommon/%(filename)s' + ':%(relfilename)s' ':%(funcName)s' '#%(lineno)s' ':%(message)s: ' ), filename=LOGFILE, level=logging.DEBUG) - debug = logging.getLogger(name=__name__).debug + logger = logging.getLogger(name=__name__) + logger.addFilter(_Debug_Filter()) + debug = logger.debug else: def debug(x): return None -- cgit v0.12 From 5b2b45478de3f3719e42be7511260d40199fac38 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Fri, 7 Aug 2020 19:38:50 -0400 Subject: Indicate that changes are internal. [ci skip] --- CHANGES.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index cfc3a02..f3900f0 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -9,8 +9,8 @@ NOTE: The 4.0.0 Release of SCons dropped Python 2.7 Support RELEASE VERSION/DATE TO BE FILLED IN LATER From Joseph Brill: - - MSVC and test updates: Rework the msvc installed versions cache so that it is not - exposed externally and update external references accordingly. + - Internal MSVC and test updates: Rework the msvc installed versions cache so that it + is not exposed externally and update external references accordingly. From William Deegan: - Fix yacc tool, not respecting YACC set at time of tool initialization. -- cgit v0.12 From a77120cbaf51b1049ffa35704d7b5eac78030b4c Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sun, 9 Aug 2020 07:24:30 -0600 Subject: Tweak new dictionary sidebar in userguide [ci skip] Signed-off-by: Mats Wichmann --- doc/user/environments.xml | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/doc/user/environments.xml b/doc/user/environments.xml index 8c492b3..99df99d 100644 --- a/doc/user/environments.xml +++ b/doc/user/environments.xml @@ -479,15 +479,16 @@ environment, of directory names, suffixes, etc. and key-value store) associates keys with values, such that asking the dict about a key gives you back the associated value and assigning to a key creates the association - either a new - setting if the key was unknown, or replacing the previous - previous association if the key was already in the dict. - Accessing can be done by using [] - indexing notation, and dictionaries also provide a + setting if the key was unknown, or replacing the + previous association if the key was already in the dictionary. + Values can be retrieved using item access + (the key name in square brackets ([])), + and dictionaries also provide a method named get which responds - with a default value (None or a default - you supply) if the key is not in the dictionary, which - avoids failing in that case. The syntax - for a dictionary itself uses curly braces ({}). + with a default value, either None or a value + you supply as the second argument, if the key is not in the dictionary, + which avoids failing in that case. The syntax + for initializing a dictionary uses curly braces ({}). Here are some simple examples (inspired by those in the official Python tutorial) using syntax that indicates interacting with the &Python; interpreter @@ -498,7 +499,7 @@ environment, of directory names, suffixes, etc. >>> tel = {'jack': 4098, 'sape': 4139} ->>> tel['guido'] = 4098 +>>> tel['guido'] = 4127 >>> tel['jack'] 4098 >>> del tel['sape'] @@ -522,7 +523,8 @@ None a &consenv; is a &Python; dictionary. The os.environ value that &Python; uses to make available the external environment is also a - dictionary. We will need these concepts in this chapter. + dictionary. We will need these concepts in this chapter + and throughout the rest of this guide. @@ -538,10 +540,10 @@ None the user has in force when executing &SCons; are available in the &Python; - os.environ - dictionary. That syntax means the environ + os.environ dictionary. + That syntax means the environ attribute of the os module. - In Python, to access contents of a module you must first + In Python, to access the contents of a module you must first import it - so you would include the import os statement to any &SConscript; file -- cgit v0.12 From d97ab435efedef58940aed901303e5e2a89f76c5 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Mon, 10 Aug 2020 12:08:25 -0400 Subject: Restore inadvertent character changes from master. [ci skip] --- CHANGES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index c1ee31f..8a8cc24 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,4 @@ - + SCons - a software construction tool -- cgit v0.12 From 0949d5bbe3056e51e45b3975c0c3a1fb36be1c14 Mon Sep 17 00:00:00 2001 From: Joachim Kuebart Date: Tue, 11 Aug 2020 08:32:48 +0200 Subject: Conditionally suppress deprecation message. Suppress missing SConscript deprecation message when must_exist is used. --- CHANGES.txt | 4 ++ SCons/Script/SConscript.py | 2 +- test/SConscript/must_exist_deprecation.py | 79 +++++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 test/SConscript/must_exist_deprecation.py diff --git a/CHANGES.txt b/CHANGES.txt index f3900f0..0bb45cf 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -28,6 +28,10 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Complete tests for Dictionary, env.keys() and env.values() for OverrideEnvironment. Enable env.setdefault() method, add tests. + From Joachim Kuebart: + - Suppress missing SConscript deprecation warning if `must_exist` is + used. + RELEASE 4.0.1 - Mon, 16 Jul 2020 16:06:40 -0700 diff --git a/SCons/Script/SConscript.py b/SCons/Script/SConscript.py index 309bed3..e7f53f5 100644 --- a/SCons/Script/SConscript.py +++ b/SCons/Script/SConscript.py @@ -172,7 +172,7 @@ def handle_missing_SConscript(f, must_exist=None): msg = "Fatal: missing SConscript '%s'" % f.get_internal_path() raise SCons.Errors.UserError(msg) - if SCons.Script._warn_missing_sconscript_deprecated: + if SCons.Script._warn_missing_sconscript_deprecated and must_exist is None: msg = "Calling missing SConscript without error is deprecated.\n" + \ "Transition by adding must_exist=0 to SConscript calls.\n" + \ "Missing SConscript '%s'" % f.get_internal_path() diff --git a/test/SConscript/must_exist_deprecation.py b/test/SConscript/must_exist_deprecation.py new file mode 100644 index 0000000..86f753b --- /dev/null +++ b/test/SConscript/must_exist_deprecation.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +''' +Test deprecation warning if must_exist flag is used in a SConscript call +''' + +import os +import TestSCons + +test = TestSCons.TestSCons() + +# catch the exception if is raised, send it on as a warning +# this gives us traceability of the line responsible +SConstruct_path = test.workpath('SConstruct') +test.write(SConstruct_path, """\ +import SCons +from SCons.Warnings import _warningOut +import sys + +DefaultEnvironment(tools=[]) +# 1. call should succeed without deprecation warning +try: + SConscript('missing/SConscript', must_exist=False) +except SCons.Errors.UserError as e: + if _warningOut: + _warningOut(e) +# 2. call should succeed with deprecation warning +try: + SConscript('missing/SConscript') +except SCons.Errors.UserError as e: + if _warningOut: + _warningOut(e) +""") + +# we should see two warnings, the second being the deprecation message. +# need to build the path in the expected msg in an OS-agnostic way +missing = os.path.normpath('missing/SConscript') +warn1 = """ +scons: warning: Ignoring missing SConscript '{}' +""".format(missing) + test.python_file_line(SConstruct_path, 8) +warn2 = """ +scons: warning: Calling missing SConscript without error is deprecated. +Transition by adding must_exist=0 to SConscript calls. +Missing SConscript '{}' +""".format(missing) + test.python_file_line(SConstruct_path, 14) + +expect_stderr = warn1 + warn2 +test.run(arguments = ".", stderr = expect_stderr) +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: -- cgit v0.12 From 186736d72074ba06c962900bcd8c2d49fd5e828d Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Tue, 11 Aug 2020 08:22:03 -0600 Subject: Tweak Installing Python section in User Guide [skip appveyor] Signed-off-by: Mats Wichmann --- doc/user/build-install.xml | 49 ++++++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/doc/user/build-install.xml b/doc/user/build-install.xml index 87a86aa..c106dc5 100644 --- a/doc/user/build-install.xml +++ b/doc/user/build-install.xml @@ -110,12 +110,13 @@ Python 3.7.1 - Note to Windows users: there are many different ways Python - can get installed or invoked on Windows, it is beyond the scope - of this guide to unravel all of them. Try using the - Python launcher (see - PEP 397) - by using the name py instead of + Note to Windows users: there are a number of different ways Python + can be installed or invoked on Windows, it is beyond the scope + of this guide to unravel all of them. Many will have an additional + program called the Python launcher (described, + somewhat technically, in + PEP 397): + try using the command name py instead of python, if that is not available drop back to trying python. @@ -129,21 +130,23 @@ Python 3.7.1 If Python is not installed on your system, or is not findable in the current search path, you will see an error message - stating something like "command not found" + stating something like "command not found" (on UNIX or Linux) - or "'python' is not recognized - as an internal or external command, operable progam or batch file" + or "'python' is not recognized as an internal + or external command, operable progam or batch file" (on Windows cmd). - In that case, you need to install Python - (or fix the search path) + In that case, you need to either install Python + or fix the search path before you can install &SCons;. - The canonical location for information - about downloading and installing Python is - http://www.python.org/download/. - See that page and associated links to get started. + The canonical location for downloading Python + from Python's own website is: + https://www.python.org/download. + There are useful system-specific entries on setup and + usage to be found at: + https://docs.python.org/3/using @@ -153,7 +156,7 @@ Python 3.7.1 by other means, and is easier than installing from source code. Many such systems have separate packages for Python 2 and Python 3 - make sure the Python 3 package is - installed, as &SCons; requires it. + installed, as the latest &SCons; requires it. Building from source may still be a useful option if you need a version that is not offered by the distribution you are using. @@ -195,7 +198,7 @@ Python 3.7.1 For those users using Anaconda or Miniconda, use the conda installer instead, so the &scons; install location will match the version of Python that - system will be using: + system will be using. For example: @@ -211,14 +214,14 @@ Python 3.7.1 During the still-ongoing Python 2 to 3 transition, some distributions may still have two &SCons; packages available, one which uses Python 2 and one which uses Python 3. Since - latest &scons; only runs on Python 3, to get the current version + the latest &scons; only runs on Python 3, to get the current version you should choose the Python 3 package. If you need a specific version of &SCons; that is different from the package available, - pip has a version option or you can follow + pip has a version option or you can follow the instructions in the next section. @@ -229,9 +232,9 @@ Python 3.7.1 If a pre-built &SCons; package is not available for your system, - and installing using pip is not suitable, + and installing using pip is not suitable, then you can still easily build and install &SCons; using the native - Python distutils package. + Python setuptools package. @@ -265,11 +268,11 @@ Python 3.7.1 install the &scons; script in the python which is used to run the setup.py's scripts directory (/usr/local/bin or - C:\Python27\Scripts), + C:\Python37\Scripts), and will install the &SCons; build engine in the corresponding library directory for the python used (/usr/local/lib/scons or - C:\Python27\scons). + C:\Python37\scons). Because these are system directories, you may need root (on Linux or UNIX) or Administrator (on Windows) privileges to install &SCons; like this. -- cgit v0.12 From a8d13e28332585b5c9e83e85ae424d53b387b154 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Tue, 11 Aug 2020 11:09:33 -0700 Subject: Add check for pre-specified YACC to yacc tool's exist() --- SCons/Tool/yacc.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/SCons/Tool/yacc.py b/SCons/Tool/yacc.py index c7bdf4c..ab019b2 100644 --- a/SCons/Tool/yacc.py +++ b/SCons/Tool/yacc.py @@ -162,6 +162,9 @@ def generate(env): def exists(env): + if 'YACC' in env: + return env.Detect(env['YACC']) + if sys.platform == 'win32': return get_yacc_path(env) else: -- cgit v0.12 From 5fa8009133ed8bd61828fafae4f0a14869fc8c23 Mon Sep 17 00:00:00 2001 From: Joachim Kuebart Date: Tue, 11 Aug 2020 20:50:31 +0200 Subject: Convert new e2e test to file fixture. --- test/SConscript/fixture/SConstruct | 17 +++++++++++++++++ test/SConscript/must_exist_deprecation.py | 20 +------------------- 2 files changed, 18 insertions(+), 19 deletions(-) create mode 100644 test/SConscript/fixture/SConstruct diff --git a/test/SConscript/fixture/SConstruct b/test/SConscript/fixture/SConstruct new file mode 100644 index 0000000..a955efc --- /dev/null +++ b/test/SConscript/fixture/SConstruct @@ -0,0 +1,17 @@ +import SCons +from SCons.Warnings import _warningOut +import sys + +DefaultEnvironment(tools=[]) +# 1. call should succeed without deprecation warning +try: + SConscript('missing/SConscript', must_exist=False) +except SCons.Errors.UserError as e: + if _warningOut: + _warningOut(e) +# 2. call should succeed with deprecation warning +try: + SConscript('missing/SConscript') +except SCons.Errors.UserError as e: + if _warningOut: + _warningOut(e) diff --git a/test/SConscript/must_exist_deprecation.py b/test/SConscript/must_exist_deprecation.py index 86f753b..8b267ce 100644 --- a/test/SConscript/must_exist_deprecation.py +++ b/test/SConscript/must_exist_deprecation.py @@ -36,25 +36,7 @@ test = TestSCons.TestSCons() # catch the exception if is raised, send it on as a warning # this gives us traceability of the line responsible SConstruct_path = test.workpath('SConstruct') -test.write(SConstruct_path, """\ -import SCons -from SCons.Warnings import _warningOut -import sys - -DefaultEnvironment(tools=[]) -# 1. call should succeed without deprecation warning -try: - SConscript('missing/SConscript', must_exist=False) -except SCons.Errors.UserError as e: - if _warningOut: - _warningOut(e) -# 2. call should succeed with deprecation warning -try: - SConscript('missing/SConscript') -except SCons.Errors.UserError as e: - if _warningOut: - _warningOut(e) -""") +test.file_fixture("fixture/SConstruct") # we should see two warnings, the second being the deprecation message. # need to build the path in the expected msg in an OS-agnostic way -- cgit v0.12 From 690126b9f148b5b6e3f1b62f95283cf262e52dbc Mon Sep 17 00:00:00 2001 From: Joachim Kuebart Date: Tue, 11 Aug 2020 21:00:50 +0200 Subject: Clarify change description. --- CHANGES.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 0bb45cf..b9835c9 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -29,8 +29,8 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER OverrideEnvironment. Enable env.setdefault() method, add tests. From Joachim Kuebart: - - Suppress missing SConscript deprecation warning if `must_exist` is - used. + - Suppress missing SConscript deprecation warning if `must_exist=False` + is used. RELEASE 4.0.1 - Mon, 16 Jul 2020 16:06:40 -0700 -- cgit v0.12 From bac92fa6f1b86ae5667a870a87a07070a37bc69e Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Mon, 10 Aug 2020 08:15:15 -0600 Subject: manpage tweaks [skip appveyor] In DESCRIPTION, turn function references into clickable links. Also turn special variables into link references, which necessitated defining identifiers for them, as these are not part of the generated entities process. Introduce the term "execution environment" in the intro section for terminology consistency, since this term is defined in the User Guide (Chapter 7, Environments). Change wording of the -c option slightly. Signed-off-by: Mats Wichmann --- doc/man/scons.xml | 80 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 42 insertions(+), 38 deletions(-) diff --git a/doc/man/scons.xml b/doc/man/scons.xml index 5cad5c7..3d04ead 100644 --- a/doc/man/scons.xml +++ b/doc/man/scons.xml @@ -111,14 +111,15 @@ and, if necessary, the rules to build those files. Premade rules exist for building many common software components such as executable programs, object files, libraries, so that for many software projects, -only the target and input files need be specified. +only the target and input files (sources) +need be specified. When invoked, &scons; searches for a file named &SConstruct; (it also checks alternate spellings -&Sconstruct;, &sconstruct;, &SConstruct.py; &Sconstruct.py; +&Sconstruct;, &sconstruct;, &SConstruct.py;, &Sconstruct.py; and &sconstruct.py; in that order) in the current directory and reads its configuration from that file. @@ -214,12 +215,10 @@ that you want to use to build your target files are not in standard system locations, &scons; will not find them unless -you explicitly include the locations into the value of -PATH in the ENV -variable in the internal &consenv;. -Whenever you create a &consenv;, -you can propagate the value of PATH -from your external environment as follows: +you explicitly include the locations into the +execution environment by setting the path in the +ENV &consvar; in the +internal &consenv;: import os @@ -229,13 +228,18 @@ env = Environment(ENV={'PATH': os.environ['PATH']}) Similarly, if the commands use specific external environment variables that &scons; does not recognize, they can be propagated into -the internal environment: +the execution environment: import os -env = Environment(ENV={'PATH': os.environ['PATH'], - 'ANDROID_HOME': os.environ['ANDROID_HOME'], - 'ANDROID_NDK_HOME': os.environ['ANDROID_NDK_HOME']}) + +env = Environment( + ENV={ + 'PATH': os.environ['PATH'], + 'ANDROID_HOME': os.environ['ANDROID_HOME'], + 'ANDROID_NDK_HOME': os.environ['ANDROID_NDK_HOME'], + } +) Or you may explicitly propagate the invoking user's @@ -269,9 +273,9 @@ ability to define new scanners for unknown input file types. is normally executed in a top-level directory containing an &SConstruct; file. When &scons; is invoked, -the command line (including the contents -of the &SCONSFLAGS; environment variable, -if set) is processed. +the command line (including the contents of the +&SCONSFLAGS; +environment variable, if set) is processed. Command-line options (see ) are consumed. Any variable argument assignments are collected, and remaining arguments are taken as the targets to build. @@ -283,8 +287,8 @@ may be specified on the command line: scons debug=1 -These variables are available -through the &ARGUMENTS; dictionary, +These variables are available through the +&ARGUMENTS; dictionary, and can be used in the SConscript files to modify the build in any way: @@ -296,7 +300,7 @@ else: The command-line variable arguments are also available -in the &ARGLIST; list, +in the &ARGLIST; list, indexed by their order on the command line. This allows you to process them in order rather than by name, if necessary. Each &ARGLIST; entry is a tuple containing @@ -304,19 +308,19 @@ if necessary. Each &ARGLIST; entry is a tuple containing Targets on the command line may be files, directories, -or phony targets defined using the &Alias; function. +or phony targets defined using the &f-link-Alias; function. The command line targets are made available in the -&COMMAND_LINE_TARGETS; list. +&COMMAND_LINE_TARGETS; list. If no targets are specified on the command line, &scons; will build the default targets. The default targets are those specified in the SConscript files via calls -to the &Default; function; if none, the default targets are +to the &f-link-Default; function; if none, the default targets are those target files in or below the current directory. Targets specified via the &Default; function are available -in the &DEFAULT_TARGETS; list. +in the &DEFAULT_TARGETS; list. To ignore the default targets specified @@ -394,11 +398,11 @@ and export. Additional files or directories to remove can be specified using the -&Clean; function in the SConscript files. +&f-link-Clean; function in the SConscript files. Conversely, targets that would normally be removed by the invocation can be retained by calling the -&NoClean; function with those targets. +&f-link-NoClean; function with those targets. &scons; supports building multiple targets in parallel via a @@ -414,7 +418,7 @@ of simultaneous tasks that may be spawned: &scons; can maintain a cache of target (derived) files that can -be shared between multiple builds. When caching is enabled in a +be shared between multiple builds. When derived-file caching is enabled in an SConscript file, any target files built by &scons; will be copied @@ -511,11 +515,11 @@ and many of those supported by cons. -Clean up by removing all target files for which a construction -command is specified. -Also remove any files or directories associated to the construction command -using the &Clean; function. -Will not remove any targets specified by the &NoClean; function. +Clean up by removing the specified targets and their +dependencies, as well as any files or directories associated +with the specified targets through calls to the &f-link-Clean; function. +Will not remove any targets specified by the &f-link-NoClean; function. + @@ -2152,7 +2156,7 @@ but setting it after the &consenv; is constructed has no effect. As a convenience, &consvars; may also be set or modified by the parse_flags -keyword argument during object creation, +keyword argument during object creation, which has the effect of the &f-link-env-MergeFlags; method being applied to the argument value @@ -2681,7 +2685,7 @@ see the descriptions below of these variables for more information. The optional parse_flags keyword argument is recognized by builders. -This works similarly to the +This works similarly to the &f-link-env-MergeFlags; method, where the argument value is broken into individual settings and merged into the appropriate &consvars;. @@ -3072,7 +3076,7 @@ that can be used in SConscript files to affect how you want the build to be performed. - + &ARGLIST; A list of the @@ -3104,7 +3108,7 @@ for key, value in ARGLIST: - + &ARGUMENTS; A dictionary of all the @@ -3129,7 +3133,7 @@ else: - + &BUILD_TARGETS; A list of the targets which @@ -3175,7 +3179,7 @@ if 'special/program' in BUILD_TARGETS: - + &COMMAND_LINE_TARGETS; A list of the targets explicitly specified on @@ -3198,7 +3202,7 @@ if 'special/program' in COMMAND_LINE_TARGETS: - + &DEFAULT_TARGETS; A list of the target @@ -7464,7 +7468,7 @@ release, it may be necessary to specify - + SCONSFLAGS A string of options that will be used by &scons; -- cgit v0.12 From aac831b3042a63fc21aa8eb4fbaa240d30baf94b Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Wed, 12 Aug 2020 10:18:52 -0600 Subject: [PR #3772] add definition of selected targets [skip appveyor] Signed-off-by: Mats Wichmann --- doc/man/scons.xml | 79 +++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 59 insertions(+), 20 deletions(-) diff --git a/doc/man/scons.xml b/doc/man/scons.xml index 3d04ead..073b0d0 100644 --- a/doc/man/scons.xml +++ b/doc/man/scons.xml @@ -109,7 +109,7 @@ commands to build them. which specifies the files to be built (targets), and, if necessary, the rules to build those files. Premade rules exist for building many common software components -such as executable programs, object files, libraries, +such as executable programs, object files and libraries, so that for many software projects, only the target and input files (sources) need be specified. @@ -277,8 +277,10 @@ the command line (including the contents of the &SCONSFLAGS; environment variable, if set) is processed. Command-line options (see ) are consumed. -Any variable argument assignments are collected, and -remaining arguments are taken as the targets to build. +Any variable argument assignments +(see ) +are collected, and +remaining arguments are taken as targets to build. Values of variables to be passed to the SConscript files may be specified on the command line: @@ -307,21 +309,55 @@ if necessary. Each &ARGLIST; entry is a tuple containing (argname, argvalue). -Targets on the command line may be files, directories, +&SCons; acts on the selected targets, +whether the requested operation is build, no-exec or clean. +Targets are selected as follows: + + + + +Targets specified on the command line. +These may be files, directories, or phony targets defined using the &f-link-Alias; function. -The command line targets are made available in the +Directory targets are scanned by &scons; for any targets +that may be found with a destination in or under that directory. +The targets listed on the command line are made available in the &COMMAND_LINE_TARGETS; list. - + + If no targets are specified on the command line, -&scons; -will build the default targets. The default targets -are those specified in the SConscript files via calls -to the &f-link-Default; function; if none, the default targets are -those target files in or below the current directory. -Targets specified via the &Default; function are available -in the &DEFAULT_TARGETS; list. +&scons; will select those targets +specified in the SConscript files via calls +to the &f-link-Default; function. These are +known as the default targets, +and are made available in the +&DEFAULT_TARGETS; list. + + + +If there are no targets from the previous steps, +&scons; selects the current directory for scanning, +unless command-line options which affect the target +scan are detected (, +, , ). +Since targets thus selected were not the result of +user instructions, this target list is not made available +for direct inspection; use the +option if they need to be examined. + + + + +&scons; always adds to the selected targets any intermediate +targets which are necessary to build the specified ones. +For example, if constructing a shared library or dll from C +source files, &scons; will also build the object files which +will make up the library. + + + To ignore the default targets specified through calls to &Default; and instead build all @@ -377,11 +413,12 @@ also the related and options): requested, as &scons; needs to make sure any dependent files are built. -Specifying "cleanup" targets in SConscript files is not usually necessary. +Specifying "cleanup" targets in SConscript files is +usually not necessary. The -flag removes all files -necessary to build the specified target: +flag removes all selected targets: + scons -c . @@ -515,10 +552,11 @@ and many of those supported by cons. -Clean up by removing the specified targets and their -dependencies, as well as any files or directories associated -with the specified targets through calls to the &f-link-Clean; function. -Will not remove any targets specified by the &f-link-NoClean; function. +Clean up by removing the selected targets, +well as any files or directories associated +with a selected target through calls to the &f-link-Clean; function. +Will not remove any targets which are marked for +preservation through calls to the &f-link-NoClean; function. @@ -1449,6 +1487,7 @@ be appropriate for most uses. , + , , , -- cgit v0.12 From 1b2e8e9e8b5aaa938ef78a5d29e31ad4b514746a Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 10 Aug 2020 10:37:31 -0500 Subject: fix issue where java parses class incorrectly from lambdas after new --- CHANGES.txt | 3 +++ SCons/Tool/JavaCommon.py | 3 ++- SCons/Tool/JavaCommonTests.py | 41 +++++++++++++++++++++++++++++++++++++++-- 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index b9835c9..fe45dc9 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -24,6 +24,9 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER like zip(["test.zip"], ["zip_scons.py"]) and ignore ZIPCOMSTR if ZIPCOM and ZIPCOMSTR weren't set after the Environment/Tool is initialized. (Explained in PR #3659) + From Daniel Moody: + - Fix issue where java parsed a class incorrectly from lambdas used after a new. + From Mats Wichmann: - Complete tests for Dictionary, env.keys() and env.values() for OverrideEnvironment. Enable env.setdefault() method, add tests. diff --git a/SCons/Tool/JavaCommon.py b/SCons/Tool/JavaCommon.py index bb05977..d869b38 100644 --- a/SCons/Tool/JavaCommon.py +++ b/SCons/Tool/JavaCommon.py @@ -87,9 +87,10 @@ if java_parsing: # any alphanumeric token surrounded by angle brackets (generics); # the multi-line comment begin and end tokens /* and */; # array declarations "[]". + # Lambda function symbols: -> _reToken = re.compile(r'(\n|\\\\|//|\\[\'"]|[\'"{\};.()]|' + r'\d*\.\d*|[A-Za-z_][\w$.]*|<[A-Za-z_]\w+>|' + - r'/\*|\*/|\[\])') + r'/\*|\*/|\[\]|->)') class OuterState: diff --git a/SCons/Tool/JavaCommonTests.py b/SCons/Tool/JavaCommonTests.py index b0a788e..83354b8 100644 --- a/SCons/Tool/JavaCommonTests.py +++ b/SCons/Tool/JavaCommonTests.py @@ -97,7 +97,7 @@ public class Foo """Test class names with $ in them""" input = """\ -public class BadDep { +public class BadDep { public void new$rand () {} } """ @@ -484,6 +484,43 @@ public class NestedExample expect = [ 'NestedExample$1', 'NestedExample$1$1', 'NestedExample' ] assert expect == classes, (expect, classes) + def test_lambda_after_new(self): + """Test lamdas after new""" + + input = """\ +// import java.util.*; + +public class LamdaExample +{ + + public void testFunc (int arg1, String arg2, Runnable lambda){ + } + public LamdaExample() + { + testFunc( + 5, + new String("test"), + // Lambda symbol is after new, and used curly braces so + // we should not parse this as a new class. + () -> {} + ); + } + + + public static void main(String argv[]) + { + LamdaExample e = new LamdaExample(); + } +} +""" + pkg_dir, classes = SCons.Tool.JavaCommon.parse_java(input, '1.4') + expect = [ 'LamdaExample' ] + assert expect == classes, (expect, classes) + + pkg_dir, classes = SCons.Tool.JavaCommon.parse_java(input, '1.8') + expect = [ 'LamdaExample' ] + assert expect == classes, (expect, classes) + def test_private_inner_class_instantiation(self): """Test anonymous inner class generated by private instantiation""" @@ -532,7 +569,7 @@ class Broken * Detected. */ class InnerOK { InnerOK () { } } - + { System.out.println("a number: " + 1000.0 + ""); } -- cgit v0.12 From 1f726753e6776b6b9e0af8284f465dbeabaa2d53 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sat, 15 Aug 2020 08:30:16 -0600 Subject: man: expand explanations on selecting Tool modules [ci skip] Signed-off-by: Mats Wichmann --- doc/man/scons.xml | 90 +++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 58 insertions(+), 32 deletions(-) diff --git a/doc/man/scons.xml b/doc/man/scons.xml index 073b0d0..407118b 100644 --- a/doc/man/scons.xml +++ b/doc/man/scons.xml @@ -148,7 +148,10 @@ regardless of the actual file names or number of such files. looks for a directory named site_scons in various system directories and in the directory containing the -&SConstruct; file and prepends the ones it +&SConstruct; file +or, if specified, the +directory from the option instead, +and prepends the ones it finds to the Python module search path (sys.path), thus allowing modules in such directories to be imported in the normal Python way in SConscript files. @@ -2302,31 +2305,32 @@ See for details. Tools -&SCons; has a large number of predefined tools which -are used to help initialize the &consenv;, -and additional tools can be added. -An &scons; tool specification -is only responsible for setup. +&SCons; has a large number of predefined tools +(more properly, tool specifications) +which are used to help initialize the &consenv;. +An &scons; tool is only responsible for setup. For example, if the SConscript file declares the need to construct an object file from a C-language source file by calling the &b-link-Object; builder, then a tool representing an available C compiler needs to have run first, to set up the builder and all the &consvars; -it needs, in that &consenv;. Normally this +it needs in the associated &consenv;; the tool itself +is not called in the process of the build. Normally this happens invisibly: &scons; has per-platform lists of default tools, and it runs through those tools, -calling the ones which are actually applicable -(skipping those where necessary programs are not -installed on the build system, etc.). +calling the ones which are actually applicable, +skipping those where necessary programs are not +installed on the build system, or other preconditions are not met. A specific set of tools -with which to initialize the environment when +with which to initialize an environment when creating it may be specified using the optional keyword argument -tools. +tools, which takes a list +of tool names. This is useful to override the defaults, to specify non-default built-in tools, and to supply added tools: @@ -2336,7 +2340,7 @@ env = Environment(tools=['msvc', 'lex']) -Tools can also be called by using the &f-link-Tool; +Tools can also be directly called by using the &f-link-Tool; method (see below). @@ -2351,39 +2355,48 @@ The tool name 'default' can be used to retain the default list. -If no tools list is specified, -or the list includes 'default', -then &scons; will detect usable tools, -using the value of PATH -in the ENV &consvar; (not -the external PATH from os.environ) +If no tools argument is specified, +or if tools includes 'default', +then &scons; will auto-detect usable tools, +using the execution environment value of PATH +(that is, env['ENV']['PATH'] - +the external evironment PATH from os.environ +is not used) for looking up any backing programs, and the platform name in effect to determine the default tools for that platform. Changing the PATH variable after the &consenv; is constructed will not cause the tools to -be redetected. +be re-detected. -To help locate added tools, specify the -toolpath keyword argument: + +Additional tools can be added to a project either by +placing them in a site_tools subdirectory +of a site directory, or in a custom location specified to +&scons; by giving the +toolpath keyword argument. +toolpath also takes a list as its value: + env = Environment(tools=['default', 'foo'], toolpath=['tools']) -This looks for a tool specification in tools/foo.py +This looks for a tool specification module foo.py +in directory tools and in the standard locations, as well as using the ordinary default tools for the platform. -Tools in the toolpath are used in preference to -any of the built-in ones. For example, adding -a tool gcc.py to the toolpath -directory would override the built-in gcc tool. +Tools in a specified toolpath are used in preference to any other location; +tools in a site_tools directory are also +take precedence over built-in ones. For example, adding +a tool specification module gcc.py to the toolpath +directory would override the built-in &t-link-gcc; tool. The toolpath is stored in the environment and will be -picked up by subsequent calls to the -&f-Clone; and &f-Tool; methods: +used by subsequent calls to the &f-link-Tool; method, +as well as by &f-link-env-Clone;. @@ -2393,7 +2406,7 @@ derived.CustomBuilder() -A tool specification must include two functions: +A tool specification module must include two functions: @@ -2412,7 +2425,8 @@ to vary its initialization. exists(env) Return True if the tool can -be called. Usually this means looking up one or more +be called in the context of env. +Usually this means looking up one or more known programs using the PATH from the supplied env, but the tool can make the "exists" decision in any way it chooses. @@ -2421,6 +2435,18 @@ make the "exists" decision in any way it chooses. + + +At the moment, user-added tools do not automatically have their +exists function called. +As a result, it is recommended that the generate +function be defensively coded - that is, do not rely on any +necessary existence checks already having been performed. +This is expected to be a temporary limitation, +and the exists function should still be provided. + + + The elements of the tools list may also be functions or callable objects, in which case the &Environment; method @@ -6906,7 +6932,7 @@ script named The MinGW bin directory must be in your PATH environment variable or the -ENV['PATH'] &consvar; for &scons; +['ENV']['PATH'] &consvar; for &scons; to detect and use the MinGW tools. When running under the native Windows Python interpreter, &scons; will prefer the MinGW tools over the Cygwin tools, if they are both installed, regardless of the order of the bin -- cgit v0.12 From 28c58aaa3d34a37723a7d475495eb763ba51d009 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Mon, 17 Aug 2020 08:20:51 -0600 Subject: Update docs on documentation [ci skip] Signed-off-by: Mats Wichmann --- bin/SConsDoc.py | 36 ++++++++++++++++++------ doc/overview.rst | 86 +++++++++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 96 insertions(+), 26 deletions(-) diff --git a/bin/SConsDoc.py b/bin/SConsDoc.py index ba9c923..baffbdc 100644 --- a/bin/SConsDoc.py +++ b/bin/SConsDoc.py @@ -41,9 +41,11 @@ Builder example: This is the summary description of an SCons Builder. It will get placed in the man page, and in the appropriate User's Guide appendix. - The name of any builder may be interpolated + The name of this builder may be interpolated anywhere in the document by specifying the - &b-BUILDER; element. It need not be on a line by itself. + &b-BUILDER; element. A link to this definition may be + interpolated by specifying the &b-link-BUILDER; element. + Unlike normal XML, blank lines are significant in these descriptions and serve to separate paragraphs. @@ -59,16 +61,28 @@ Builder example: Function example: - + (arg1, arg2, key=value) This is the summary description of an SCons function. It will get placed in the man page, and in the appropriate User's Guide appendix. - The name of any builder may be interpolated + If the "signature" attribute is specified, SIGTYPE may be one + of "global", "env" or "both" (the default if omitted is "both"), + to indicate the signature applies to the global form or the + environment form, or to generate both with the same signature + (excepting the insertion of "env."). + This allows for the cases of + describing that only one signature should be generated, + or both signatures should be generated and they differ, + or both signatures should be generated and they are the same. + The name of this function may be interpolated anywhere in the document by specifying the - &f-FUNCTION; element. It need not be on a line by itself. + &f-FUNCTION; element or the &f-env-FUNCTION; element. + Links to this definition may be interpolated by specifying + the &f-link-FUNCTION: or &f-link-env-FUNCTION; element. + print("this is example code, it will be offset and indented") @@ -83,9 +97,11 @@ Construction variable example: This is the summary description of a construction variable. It will get placed in the man page, and in the appropriate User's Guide appendix. - The name of any construction variable may be interpolated + The name of this construction variable may be interpolated anywhere in the document by specifying the - &t-VARIABLE; element. It need not be on a line by itself. + &cv-VARIABLE; element. A link to this definition may be + interpolated by specifying the &cv-link-VARIABLE; element. + print("this is example code, it will be offset and indented") @@ -100,9 +116,11 @@ Tool example: This is the summary description of an SCons Tool. It will get placed in the man page, and in the appropriate User's Guide appendix. - The name of any tool may be interpolated + The name of this tool may be interpolated anywhere in the document by specifying the - &t-TOOL; element. It need not be on a line by itself. + &t-TOOL; element. A link to this definition may be + interpolated by specifying the &t-link-TOOL; element. + print("this is example code, it will be offset and indented") diff --git a/doc/overview.rst b/doc/overview.rst index 9a2558a..74aa688 100644 --- a/doc/overview.rst +++ b/doc/overview.rst @@ -92,12 +92,19 @@ Entities ======== We are using entities for special keywords like ``SCons`` that should -appear with the same formatting throughout the text. These are kept in -a single file ``doc/scons.mod`` which gets included by the documents. - -Additionally, for each Tool, Builder, Cvar (construction variable) and -Function, a bunch of linkends in the form of entities get defined. They -can be used in the MAN page and the User manual. +appear with the same formatting throughout the text. This allows a +single place to make styling changes if needed. These are kept in +a single file ``doc/scons.mod`` which gets included by the documents, +and can be used anywhere in the documentation files. + +Additionally, for the definitions of the four special types available +in the SCons doctype - Tool, Builder, Construction Variable and Function - +a bunch of reference links in the form of entities are generated. +These entities can be used in the MAN page and the User manual. +Note that the four type tags themselves (````, ````, +```` and ````) can only be used in documentation +sources in the ``SCons`` directory; the build will not scan for these +in the ``doc`` directory. When you add an XML file in the ``SCons/Tools`` folder, e.g. for a tool named ``foobar``, you can use the two entities @@ -134,12 +141,12 @@ By calling the script :: python bin/docs-update-generated.py - + you can recreate the lists of entities (``*.mod``) in the ``generated`` -folder, if required. At the same time, this will generate the ``*.gen`` +folder. At the same time, this will generate the matching ``*.gen`` files, which list the full description of all the Builders, Tools, -Functions and CVars for the MAN page and the User Guide's appendix. -Thus, you want to regenerate when there's a change to +Functions and CVars for the MAN page and the User Guide's appendix. +Thus, you want to regenerate when there's a change to any of those four special elements, or an added or deleted element. These generated files are left checked in so in the normal case you can just rebuild the docs without having to first generate the entity @@ -150,14 +157,59 @@ refer to the start of the Python script ``bin/SConsDoc.py``. It explains the available tags and the exact syntax in detail. -Examples -======== +Linking +======= + +Normal Docbook (v4.5 style, as of this writing) in-document linking +is supported, as is linking to documents with a web address. +For any element in a document, you can include an ``id=name`` +attribute to set an identifier, and write a link to that identifier. +Many of the section headings already have such identifiers, +and it is fine to add more, as long as they remain unique. +As noted in the previous section, for the special types, +entities are generated which contain links, +so you can just use those entities instead +of writing the link reference manually. + +There is something to keep in mind about linking, however. +Cross-document links between the MAN page and the User Guide +do not work. But some text is shared between the two, which +allows the appearance of such linking, and this is where it +gets a little confusing. The text defined by the four special +types is generated into the ``*.gen`` files, +which get included both in the appropriate places in the MAN page, +and in the Appendix in the User Guide. Using entities within +this shared content is fine. Writing links in this shared +content to element identifiers defined elsewhere is not. + +That sounds a little confusing so here is a real example: +an xml source file in ``SCons`` defines the ``SCANNERS`` +construction variable by using `` ... ``. +This will generate the linking entity ``&cv-link-SCANNERS;``, +which can be used anywhere the ``doc/generated/variables.gen`` +file is included (i.e. MAN page and User Guide for now) +to leave a link to this definition. +But the text written inside the ``SCANNERS`` definition +also wants to refer to the "Builder Objects" and "Scanner +Objects" sections in the MAN page, as this contains relevant +further description. This reference should not include an +XML link, even though the MAN page defines the two identifiers +``scanner_objects`` and ``builder_objects``, because this +definition will *also* be included in the User Guide, which +has no such section names or identifiers. It is better here +to write it all in text, as in *See the manpage section +"Builder Objects"* than to leave a dangling reference in one +of the docs. + +SCons Examples +============== In the User Guide, we support automatically created examples. This means that the output of the specified source files and SConstructs is generated by running them with the current SCons version. We do this to ensure that the output displayed in the manual is identical to what -you get when you run the example on the command-line. +you get when you run the example on the command-line, without having +to remember to manually update the example outputs all the time. A short description about how these examples have to be defined can be found at the start of the file ``bin/SConsExamples.py``. Call @@ -212,17 +264,17 @@ User Guide. *generated* Entity lists and outputs of the UserGuide examples. They get generated - by the update scripts ``bin/docs-update-generated.py`` + by the update scripts ``bin/docs-update-generated.py`` and ``bin/docs-create-example-outputs.py``. *images* Images for the ``overview.rst`` document. - + *xsd* The SCons Docbook schema (XSD), based on the Docbook v4.5 DTD/XSD. - + *xslt* XSLT transformation scripts for converting the special SCons tags like ``scons_output`` to valid Docbook during document processing. - + -- cgit v0.12 From a566609d60b893e018926b178992030b783d697d Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Tue, 18 Aug 2020 11:00:10 -0600 Subject: Further tweak to tool location doc (toolpath vs. site_tools vs builtin) [ci skip] Signed-off-by: Mats Wichmann --- doc/man/scons.xml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/doc/man/scons.xml b/doc/man/scons.xml index 407118b..fc28e2a 100644 --- a/doc/man/scons.xml +++ b/doc/man/scons.xml @@ -2388,12 +2388,15 @@ as well as using the ordinary default tools for the platform. -Tools in a specified toolpath are used in preference to any other location; -tools in a site_tools directory are also -take precedence over built-in ones. For example, adding +Directories specified via toolpath are prepended +to the existing tool path. The default tool path is any site_tools +directories, so tools in a specified toolpath +take priority, +followed by tools in a site_tools directory, +followed by built-in tools. For example, adding a tool specification module gcc.py to the toolpath directory would override the built-in &t-link-gcc; tool. -The toolpath is +The tool path is stored in the environment and will be used by subsequent calls to the &f-link-Tool; method, as well as by &f-link-env-Clone;. -- cgit v0.12 From c1acf2f557a0e99ef8f97d694f346e8c10ba6b97 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sun, 23 Aug 2020 08:12:37 -0600 Subject: Reject abbreviations of AddOption options. If a cmdline option is left over, and looks like an abbreviation of an AddOption'd option, raise an error. Signed-off-by: Mats Wichmann --- CHANGES.txt | 2 ++ SCons/Script/Main.xml | 4 ++-- SCons/Script/SConsOptions.py | 11 +++++++---- test/AddOption/longopts.py | 10 ++++++++++ 4 files changed, 21 insertions(+), 6 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index fe45dc9..7719cf3 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -30,6 +30,8 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER From Mats Wichmann: - Complete tests for Dictionary, env.keys() and env.values() for OverrideEnvironment. Enable env.setdefault() method, add tests. + - Raise an error if an option (not otherwise consumed) is used which + looks like an abbreviation of one one added by AddOption. (#3653) From Joachim Kuebart: - Suppress missing SConscript deprecation warning if `must_exist=False` diff --git a/SCons/Script/Main.xml b/SCons/Script/Main.xml index 0a8f9dc..f65ffca 100644 --- a/SCons/Script/Main.xml +++ b/SCons/Script/Main.xml @@ -36,13 +36,13 @@ are the same as those supported by the add_option method in the standard Python library module optparse, with a few additional capabilities noted below. See the documentation for -optparse +optparse for a thorough discussion of its option-processing capabities. In addition to the arguments and values supported by the -optparse +optparse add_option method, &f-AddOption; allows setting the diff --git a/SCons/Script/SConsOptions.py b/SCons/Script/SConsOptions.py index 9fee74e..6e95163 100644 --- a/SCons/Script/SConsOptions.py +++ b/SCons/Script/SConsOptions.py @@ -273,8 +273,7 @@ class SConsOptionParser(optparse.OptionParser): """ arg = rargs.pop(0) - # Value explicitly attached to arg? Pretend it's the next - # argument. + # Value explicitly attached to arg? Pretend it's the next argument. if "=" in arg: (opt, next_arg) = arg.split("=", 1) rargs.insert(0, next_arg) @@ -284,7 +283,11 @@ class SConsOptionParser(optparse.OptionParser): had_explicit_value = False try: - opt = self._match_long_opt(opt) + if opt != self._match_long_opt(opt): + raise optparse.BadOptionError( + "cannot use abbreviated option %s, use %s instead" + % (opt, self._match_long_opt(opt)) + ) except optparse.BadOptionError: if self.preserve_unknown_options: # SCons-specific: if requested, add unknown options to @@ -398,7 +401,7 @@ class SConsOptionParser(optparse.OptionParser): """ Adds a local option to the parser. - This is initiated by a SetOption() call to add a user-defined + This is initiated by an AddOption() call to add a user-defined command-line option. We add the option to a separate option group for the local options, creating the group if necessary. """ diff --git a/test/AddOption/longopts.py b/test/AddOption/longopts.py index 47ae4f1..0fcaa5b 100644 --- a/test/AddOption/longopts.py +++ b/test/AddOption/longopts.py @@ -51,6 +51,16 @@ test.run('-Q -q . --myargument=helloworld', test.run('-Q -q . --myarg=helloworld', stdout="myargument: gully\nmyarg: helloworld\n") +# Issue #3653: add a check for an abbreviation which never gets AddOption'd. +test.run('-Q -q . --myargumen=helloworld', status=2, + stdout="myargument: gully\nmyarg: balla\n", + stderr="""\ +usage: scons [OPTION] [TARGET] ... + +SCons Error: no such option: cannot use abbreviated option --myargumen, use --myargument instead +""") + + test.pass_test() # Local Variables: -- cgit v0.12 From 30e9339739c411c1eb8fd514e1d0e995ddddb9b1 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Mon, 24 Aug 2020 21:20:36 -0600 Subject: [PR #3784] adjust opt-abbreviation msg after PR review Signed-off-by: Mats Wichmann --- SCons/Script/SConsOptions.py | 2 +- test/AddOption/longopts.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/SCons/Script/SConsOptions.py b/SCons/Script/SConsOptions.py index 6e95163..aa31635 100644 --- a/SCons/Script/SConsOptions.py +++ b/SCons/Script/SConsOptions.py @@ -285,7 +285,7 @@ class SConsOptionParser(optparse.OptionParser): try: if opt != self._match_long_opt(opt): raise optparse.BadOptionError( - "cannot use abbreviated option %s, use %s instead" + "'%s'. Did you mean '%s'?" % (opt, self._match_long_opt(opt)) ) except optparse.BadOptionError: diff --git a/test/AddOption/longopts.py b/test/AddOption/longopts.py index 0fcaa5b..48c3502 100644 --- a/test/AddOption/longopts.py +++ b/test/AddOption/longopts.py @@ -57,7 +57,7 @@ test.run('-Q -q . --myargumen=helloworld', status=2, stderr="""\ usage: scons [OPTION] [TARGET] ... -SCons Error: no such option: cannot use abbreviated option --myargumen, use --myargument instead +SCons Error: no such option: '--myargumen'. Did you mean '--myargument'? """) -- cgit v0.12 From a4d4834c9e93b485348236dd7fb953f9ef500bd1 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Tue, 18 Aug 2020 12:59:47 -0600 Subject: Simplify more tests to use unittest.main For issue #3113, but does not complete it - there are still uses of TestSuite where the "simple fix" from the issue requires more thought. Signed-off-by: Mats Wichmann --- SCons/BuilderTests.py | 11 +---------- SCons/EnvironmentTests.py | 21 ++++++--------------- SCons/PathListTests.py | 11 +---------- SCons/SConsignTests.py | 13 +------------ SCons/Scanner/RCTests.py | 7 ------- 5 files changed, 9 insertions(+), 54 deletions(-) diff --git a/SCons/BuilderTests.py b/SCons/BuilderTests.py index 6dfefd5..d8c9d48 100644 --- a/SCons/BuilderTests.py +++ b/SCons/BuilderTests.py @@ -1617,16 +1617,7 @@ class CompositeBuilderTestCase(unittest.TestCase): assert str(err) == expect, err if __name__ == "__main__": - suite = unittest.TestSuite() - tclasses = [ - BuilderTestCase, - CompositeBuilderTestCase - ] - for tclass in tclasses: - names = unittest.getTestCaseNames(tclass, 'test_') - suite.addTests(list(map(tclass, names))) - - TestUnit.run(suite) + unittest.main() # Local Variables: # tab-width:4 diff --git a/SCons/EnvironmentTests.py b/SCons/EnvironmentTests.py index b8f18d5..7a26a61 100644 --- a/SCons/EnvironmentTests.py +++ b/SCons/EnvironmentTests.py @@ -1376,7 +1376,7 @@ def generate(env, **kw): env[k] = v def exists(env): - return 1 + return True """) env = self.TestEnvironment(tools = [('faketool', {'a':1, 'b':2, 'c':3})], @@ -1856,14 +1856,14 @@ def exists(env): test.write('xxx.py', """\ def exists(env): - 1 + return True def generate(env): env['XXX'] = 'one' """) test.write('yyy.py', """\ def exists(env): - 1 + return True def generate(env): env['YYY'] = 'two' """) @@ -2481,14 +2481,14 @@ f5: \ test.write('xxx.py', """\ def exists(env): - 1 + return True def generate(env): env['XXX'] = 'one' """) test.write('yyy.py', """\ def exists(env): - 1 + return True def generate(env): env['YYY'] = 'two' """) @@ -3951,16 +3951,7 @@ class EnvironmentVariableTestCase(unittest.TestCase): if __name__ == "__main__": - suite = unittest.TestSuite() - tclasses = [ SubstitutionTestCase, - BaseTestCase, - OverrideEnvironmentTestCase, - NoSubstitutionProxyTestCase, - EnvironmentVariableTestCase ] - for tclass in tclasses: - names = unittest.getTestCaseNames(tclass, 'test_') - suite.addTests(list(map(tclass, names))) - TestUnit.run(suite) + unittest.main() # Local Variables: # tab-width:4 diff --git a/SCons/PathListTests.py b/SCons/PathListTests.py index 0d15c08..80f2b9c 100644 --- a/SCons/PathListTests.py +++ b/SCons/PathListTests.py @@ -189,16 +189,7 @@ class PathListTestCase(unittest.TestCase): if __name__ == "__main__": - suite = unittest.TestSuite() - tclasses = [ - subst_pathTestCase, - PathListCacheTestCase, - PathListTestCase, - ] - for tclass in tclasses: - names = unittest.getTestCaseNames(tclass, 'test_') - suite.addTests(list(map(tclass, names))) - TestUnit.run(suite) + unittest.main() # Local Variables: # tab-width:4 diff --git a/SCons/SConsignTests.py b/SCons/SConsignTests.py index 2f832b4..84c1d41 100644 --- a/SCons/SConsignTests.py +++ b/SCons/SConsignTests.py @@ -382,18 +382,7 @@ class writeTestCase(SConsignTestCase): if __name__ == "__main__": - suite = unittest.TestSuite() - tclasses = [ - BaseTestCase, - SConsignDBTestCase, - SConsignDirFileTestCase, - SConsignFileTestCase, - writeTestCase, - ] - for tclass in tclasses: - names = unittest.getTestCaseNames(tclass, 'test_') - suite.addTests(list(map(tclass, names))) - TestUnit.run(suite) + unittest.main() # Local Variables: # tab-width:4 diff --git a/SCons/Scanner/RCTests.py b/SCons/Scanner/RCTests.py index 347149c..487250e 100644 --- a/SCons/Scanner/RCTests.py +++ b/SCons/Scanner/RCTests.py @@ -158,13 +158,6 @@ class RCScannerTestCase3(unittest.TestCase): deps_match(self, deps, headers) -def suite(): - suite = unittest.TestSuite() - suite.addTest(RCScannerTestCase1()) - suite.addTest(RCScannerTestCase2()) - suite.addTest(RCScannerTestCase3()) - return suite - if __name__ == "__main__": unittest.main() -- cgit v0.12 From 20e2a0683d2f2730a3ff636ab3fd04a2676ca7a8 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Tue, 25 Aug 2020 12:39:42 -0700 Subject: Refactor shared library/module versioning logic to SCons/Tool/linkCommon/ --- SCons/Tool/__init__.py | 485 +----------------------------------- SCons/Tool/applelink.py | 3 +- SCons/Tool/cyglink.py | 131 ++++++---- SCons/Tool/dmd.py | 17 +- SCons/Tool/gdc.py | 3 +- SCons/Tool/install.py | 8 +- SCons/Tool/ldc.py | 10 +- SCons/Tool/link.py | 28 +-- SCons/Tool/linkCommon/__init__.py | 503 ++++++++++++++++++++++++++++++++++++++ 9 files changed, 616 insertions(+), 572 deletions(-) create mode 100644 SCons/Tool/linkCommon/__init__.py diff --git a/SCons/Tool/__init__.py b/SCons/Tool/__init__.py index 32017cb..24fb9d2 100644 --- a/SCons/Tool/__init__.py +++ b/SCons/Tool/__init__.py @@ -39,7 +39,6 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import sys import os -from collections.abc import Callable import importlib.util import SCons.Builder @@ -51,6 +50,8 @@ import SCons.Scanner.D import SCons.Scanner.LaTeX import SCons.Scanner.Prog import SCons.Scanner.SWIG +from SCons.Tool.linkCommon import ShLibPrefixGenerator, LdModPrefixGenerator, ShLibSuffixGenerator, \ + LdModSuffixGenerator, LibSymlinksActionFunction, LibSymlinksStrFun DefaultToolpath = [] @@ -272,6 +273,9 @@ class Tool: return self.name +LibSymlinksAction = SCons.Action.Action(LibSymlinksActionFunction, LibSymlinksStrFun) + + ########################################################################## # Create common executable program / library / object builders @@ -325,485 +329,6 @@ def createStaticLibBuilder(env): return static_lib -def _call_linker_cb(env, callback, args, result=None): - """Returns the result of env['LINKCALLBACKS'][callback](*args) - if env['LINKCALLBACKS'] is a dictionary and env['LINKCALLBACKS'][callback] - is callable. If these conditions are not met, return the value provided as - the *result* argument. This function is mainly used for generating library - info such as versioned suffixes, symlink maps, sonames etc. by delegating - the core job to callbacks configured by current linker tool""" - - Verbose = False - - if Verbose: - print('_call_linker_cb: args=%r' % args) - print('_call_linker_cb: callback=%r' % callback) - - try: - cbfun = env['LINKCALLBACKS'][callback] - except (KeyError, TypeError): - if Verbose: - print('_call_linker_cb: env["LINKCALLBACKS"][%r] not found or can not be used' % callback) - pass - else: - if Verbose: - print('_call_linker_cb: env["LINKCALLBACKS"][%r] found' % callback) - print('_call_linker_cb: env["LINKCALLBACKS"][%r]=%r' % (callback, cbfun)) - if isinstance(cbfun, Callable): - if Verbose: - print('_call_linker_cb: env["LINKCALLBACKS"][%r] is callable' % callback) - result = cbfun(env, *args) - return result - - -def _call_env_subst(env, string, *args, **kw): - kw2 = {} - for k in ('raw', 'target', 'source', 'conv', 'executor'): - try: - kw2[k] = kw[k] - except KeyError: - pass - return env.subst(string, *args, **kw2) - - -class _ShLibInfoSupport: - @property - def libtype(self): - return 'ShLib' - - def get_lib_prefix(self, env, *args, **kw): - return _call_env_subst(env, '$SHLIBPREFIX', *args, **kw) - - def get_lib_suffix(self, env, *args, **kw): - return _call_env_subst(env, '$SHLIBSUFFIX', *args, **kw) - - def get_lib_version(self, env, *args, **kw): - return _call_env_subst(env, '$SHLIBVERSION', *args, **kw) - - def get_lib_noversionsymlinks(self, env, *args, **kw): - return _call_env_subst(env, '$SHLIBNOVERSIONSYMLINKS', *args, **kw) - - -class _LdModInfoSupport: - @property - def libtype(self): - return 'LdMod' - - def get_lib_prefix(self, env, *args, **kw): - return _call_env_subst(env, '$LDMODULEPREFIX', *args, **kw) - - def get_lib_suffix(self, env, *args, **kw): - return _call_env_subst(env, '$LDMODULESUFFIX', *args, **kw) - - def get_lib_version(self, env, *args, **kw): - return _call_env_subst(env, '$LDMODULEVERSION', *args, **kw) - - def get_lib_noversionsymlinks(self, env, *args, **kw): - return _call_env_subst(env, '$LDMODULENOVERSIONSYMLINKS', *args, **kw) - - -class _ImpLibInfoSupport: - @property - def libtype(self): - return 'ImpLib' - - def get_lib_prefix(self, env, *args, **kw): - return _call_env_subst(env, '$IMPLIBPREFIX', *args, **kw) - - def get_lib_suffix(self, env, *args, **kw): - return _call_env_subst(env, '$IMPLIBSUFFIX', *args, **kw) - - def get_lib_version(self, env, *args, **kw): - version = _call_env_subst(env, '$IMPLIBVERSION', *args, **kw) - if not version: - try: - lt = kw['implib_libtype'] - except KeyError: - pass - else: - if lt == 'ShLib': - version = _call_env_subst(env, '$SHLIBVERSION', *args, **kw) - elif lt == 'LdMod': - version = _call_env_subst(env, '$LDMODULEVERSION', *args, **kw) - return version - - def get_lib_noversionsymlinks(self, env, *args, **kw): - disable = None - try: - env['IMPLIBNOVERSIONSYMLINKS'] - except KeyError: - try: - lt = kw['implib_libtype'] - except KeyError: - pass - else: - if lt == 'ShLib': - disable = _call_env_subst(env, '$SHLIBNOVERSIONSYMLINKS', *args, **kw) - elif lt == 'LdMod': - disable = _call_env_subst(env, '$LDMODULENOVERSIONSYMLINKS', *args, **kw) - else: - disable = _call_env_subst(env, '$IMPLIBNOVERSIONSYMLINKS', *args, **kw) - return disable - - -class _LibInfoGeneratorBase: - """Generator base class for library-related info such as suffixes for - versioned libraries, symlink maps, sonames etc. It handles commonities - of SharedLibrary and LoadableModule - """ - _support_classes = {'ShLib': _ShLibInfoSupport, - 'LdMod': _LdModInfoSupport, - 'ImpLib': _ImpLibInfoSupport} - - def __init__(self, libtype, infoname): - self.libtype = libtype - self.infoname = infoname - - @property - def libtype(self): - return self._support.libtype - - @libtype.setter - def libtype(self, libtype): - try: - support_class = self._support_classes[libtype] - except KeyError: - raise ValueError('unsupported libtype %r' % libtype) - self._support = support_class() - - def get_lib_prefix(self, env, *args, **kw): - return self._support.get_lib_prefix(env, *args, **kw) - - def get_lib_suffix(self, env, *args, **kw): - return self._support.get_lib_suffix(env, *args, **kw) - - def get_lib_version(self, env, *args, **kw): - return self._support.get_lib_version(env, *args, **kw) - - def get_lib_noversionsymlinks(self, env, *args, **kw): - return self._support.get_lib_noversionsymlinks(env, *args, **kw) - - def get_versioned_lib_info_generator(self, **kw): - """ - Returns name of generator linker callback that will be used to generate - our info for a versioned library. For example, if our libtype is 'ShLib' - and infoname is 'Prefix', it would return 'VersionedShLibPrefix'. - """ - try: - libtype = kw['generator_libtype'] - except KeyError: - libtype = self.libtype - return 'Versioned%s%s' % (libtype, self.infoname) - - def generate_versioned_lib_info(self, env, args, result=None, **kw): - callback = self.get_versioned_lib_info_generator(**kw) - return _call_linker_cb(env, callback, args, result) - - -class _LibPrefixGenerator(_LibInfoGeneratorBase): - """Library prefix generator, used as target_prefix in SharedLibrary and - LoadableModule builders""" - - def __init__(self, libtype): - super(_LibPrefixGenerator, self).__init__(libtype, 'Prefix') - - def __call__(self, env, sources=None, **kw): - Verbose = False - - if sources and 'source' not in kw: - kw2 = kw.copy() - kw2['source'] = sources - else: - kw2 = kw - - prefix = self.get_lib_prefix(env, **kw2) - if Verbose: - print("_LibPrefixGenerator: input prefix=%r" % prefix) - - version = self.get_lib_version(env, **kw2) - if Verbose: - print("_LibPrefixGenerator: version=%r" % version) - - if version: - prefix = self.generate_versioned_lib_info(env, [prefix, version], prefix, **kw2) - - if Verbose: - print("_LibPrefixGenerator: return prefix=%r" % prefix) - return prefix - - -ShLibPrefixGenerator = _LibPrefixGenerator('ShLib') -LdModPrefixGenerator = _LibPrefixGenerator('LdMod') -ImpLibPrefixGenerator = _LibPrefixGenerator('ImpLib') - - -class _LibSuffixGenerator(_LibInfoGeneratorBase): - """Library suffix generator, used as target_suffix in SharedLibrary and - LoadableModule builders""" - - def __init__(self, libtype): - super(_LibSuffixGenerator, self).__init__(libtype, 'Suffix') - - def __call__(self, env, sources=None, **kw): - Verbose = False - - if sources and 'source' not in kw: - kw2 = kw.copy() - kw2['source'] = sources - else: - kw2 = kw - - suffix = self.get_lib_suffix(env, **kw2) - if Verbose: - print("_LibSuffixGenerator: input suffix=%r" % suffix) - - version = self.get_lib_version(env, **kw2) - if Verbose: - print("_LibSuffixGenerator: version=%r" % version) - - if version: - suffix = self.generate_versioned_lib_info(env, [suffix, version], suffix, **kw2) - - if Verbose: - print("_LibSuffixGenerator: return suffix=%r" % suffix) - return suffix - - -ShLibSuffixGenerator = _LibSuffixGenerator('ShLib') -LdModSuffixGenerator = _LibSuffixGenerator('LdMod') -ImpLibSuffixGenerator = _LibSuffixGenerator('ImpLib') - - -class _LibSymlinkGenerator(_LibInfoGeneratorBase): - """Library symlink map generator. It generates a list of symlinks that - should be created by SharedLibrary or LoadableModule builders""" - - def __init__(self, libtype): - super(_LibSymlinkGenerator, self).__init__(libtype, 'Symlinks') - - def __call__(self, env, libnode, **kw): - Verbose = False - - if libnode and 'target' not in kw: - kw2 = kw.copy() - kw2['target'] = libnode - else: - kw2 = kw - - if Verbose: - print("_LibSymLinkGenerator: libnode=%r" % libnode.get_path()) - - symlinks = None - - version = self.get_lib_version(env, **kw2) - disable = self.get_lib_noversionsymlinks(env, **kw2) - if Verbose: - print('_LibSymlinkGenerator: version=%r' % version) - print('_LibSymlinkGenerator: disable=%r' % disable) - - if version and not disable: - prefix = self.get_lib_prefix(env, **kw2) - suffix = self.get_lib_suffix(env, **kw2) - symlinks = self.generate_versioned_lib_info(env, [libnode, version, prefix, suffix], **kw2) - - if Verbose: - print('_LibSymlinkGenerator: return symlinks=%r' % StringizeLibSymlinks(symlinks)) - return symlinks - - -ShLibSymlinkGenerator = _LibSymlinkGenerator('ShLib') -LdModSymlinkGenerator = _LibSymlinkGenerator('LdMod') -ImpLibSymlinkGenerator = _LibSymlinkGenerator('ImpLib') - - -class _LibNameGenerator(_LibInfoGeneratorBase): - """Generates "unmangled" library name from a library file node. - - Generally, it's thought to revert modifications done by prefix/suffix - generators (_LibPrefixGenerator/_LibSuffixGenerator) used by a library - builder. For example, on gnulink the suffix generator used by SharedLibrary - builder appends $SHLIBVERSION to $SHLIBSUFFIX producing node name which - ends with "$SHLIBSUFFIX.$SHLIBVERSION". Correspondingly, the implementation - of _LibNameGenerator replaces "$SHLIBSUFFIX.$SHLIBVERSION" with - "$SHLIBSUFFIX" in the node's basename. So that, if $SHLIBSUFFIX is ".so", - $SHLIBVERSION is "0.1.2" and the node path is "/foo/bar/libfoo.so.0.1.2", - the _LibNameGenerator shall return "libfoo.so". Other link tools may - implement it's own way of library name unmangling. - """ - - def __init__(self, libtype): - super(_LibNameGenerator, self).__init__(libtype, 'Name') - - def __call__(self, env, libnode, **kw): - """Returns "demangled" library name""" - Verbose = False - - if libnode and 'target' not in kw: - kw2 = kw.copy() - kw2['target'] = libnode - else: - kw2 = kw - - if Verbose: - print("_LibNameGenerator: libnode=%r" % libnode.get_path()) - - version = self.get_lib_version(env, **kw2) - if Verbose: - print('_LibNameGenerator: version=%r' % version) - - name = None - if version: - prefix = self.get_lib_prefix(env, **kw2) - suffix = self.get_lib_suffix(env, **kw2) - name = self.generate_versioned_lib_info(env, [libnode, version, prefix, suffix], **kw2) - - if not name: - name = os.path.basename(libnode.get_path()) - - if Verbose: - print('_LibNameGenerator: return name=%r' % name) - - return name - - -ShLibNameGenerator = _LibNameGenerator('ShLib') -LdModNameGenerator = _LibNameGenerator('LdMod') -ImpLibNameGenerator = _LibNameGenerator('ImpLib') - - -class _LibSonameGenerator(_LibInfoGeneratorBase): - """Library soname generator. Returns library soname (e.g. libfoo.so.0) for - a given node (e.g. /foo/bar/libfoo.so.0.1.2)""" - - def __init__(self, libtype): - super(_LibSonameGenerator, self).__init__(libtype, 'Soname') - - def __call__(self, env, libnode, **kw): - """Returns a SONAME based on a shared library's node path""" - Verbose = False - - if libnode and 'target' not in kw: - kw2 = kw.copy() - kw2['target'] = libnode - else: - kw2 = kw - - if Verbose: - print("_LibSonameGenerator: libnode=%r" % libnode.get_path()) - - soname = _call_env_subst(env, '$SONAME', **kw2) - if not soname: - version = self.get_lib_version(env, **kw2) - if Verbose: - print("_LibSonameGenerator: version=%r" % version) - if version: - prefix = self.get_lib_prefix(env, **kw2) - suffix = self.get_lib_suffix(env, **kw2) - soname = self.generate_versioned_lib_info(env, [libnode, version, prefix, suffix], **kw2) - - if not soname: - # fallback to library name (as returned by appropriate _LibNameGenerator) - soname = _LibNameGenerator(self.libtype)(env, libnode) - if Verbose: - print("_LibSonameGenerator: FALLBACK: soname=%r" % soname) - - if Verbose: - print("_LibSonameGenerator: return soname=%r" % soname) - - return soname - - -ShLibSonameGenerator = _LibSonameGenerator('ShLib') -LdModSonameGenerator = _LibSonameGenerator('LdMod') - - -def StringizeLibSymlinks(symlinks): - """Converts list with pairs of nodes to list with pairs of node paths - (strings). Used mainly for debugging.""" - if SCons.Util.is_List(symlinks): - try: - return [(k.get_path(), v.get_path()) for k, v in symlinks] - except (TypeError, ValueError): - return symlinks - else: - return symlinks - - -def EmitLibSymlinks(env, symlinks, libnode, **kw): - """Used by emitters to handle (shared/versioned) library symlinks""" - Verbose = False - - # nodes involved in process... all symlinks + library - nodes = list(set([x for x, y in symlinks] + [libnode])) - - clean_targets = kw.get('clean_targets', []) - if not SCons.Util.is_List(clean_targets): - clean_targets = [clean_targets] - - for link, linktgt in symlinks: - env.SideEffect(link, linktgt) - if Verbose: - print("EmitLibSymlinks: SideEffect(%r,%r)" % (link.get_path(), linktgt.get_path())) - clean_list = [x for x in nodes if x != linktgt] - env.Clean(list(set([linktgt] + clean_targets)), clean_list) - if Verbose: - print("EmitLibSymlinks: Clean(%r,%r)" % (linktgt.get_path(), [x.get_path() for x in clean_list])) - - -def CreateLibSymlinks(env, symlinks): - """Physically creates symlinks. The symlinks argument must be a list in - form [ (link, linktarget), ... ], where link and linktarget are SCons - nodes. - """ - - Verbose = False - for link, linktgt in symlinks: - linktgt = link.get_dir().rel_path(linktgt) - link = link.get_path() - if Verbose: - print("CreateLibSymlinks: preparing to add symlink %r -> %r" % (link, linktgt)) - # Delete the (previously created) symlink if exists. Let only symlinks - # to be deleted to prevent accidental deletion of source files... - if env.fs.islink(link): - env.fs.unlink(link) - if Verbose: - print("CreateLibSymlinks: removed old symlink %r" % link) - # If a file or directory exists with the same name as link, an OSError - # will be thrown, which should be enough, I think. - env.fs.symlink(linktgt, link) - if Verbose: - print("CreateLibSymlinks: add symlink %r -> %r" % (link, linktgt)) - return 0 - - -def LibSymlinksActionFunction(target, source, env): - for tgt in target: - symlinks = getattr(getattr(tgt, 'attributes', None), 'shliblinks', None) - if symlinks: - CreateLibSymlinks(env, symlinks) - return 0 - - -def LibSymlinksStrFun(target, source, env, *args): - cmd = None - for tgt in target: - symlinks = getattr(getattr(tgt, 'attributes', None), 'shliblinks', None) - if symlinks: - if cmd is None: cmd = "" - if cmd: cmd += "\n" - cmd += "Create symlinks for: %r" % tgt.get_path() - try: - linkstr = ', '.join(["%r->%r" % (k, v) for k, v in StringizeLibSymlinks(symlinks)]) - except (KeyError, ValueError): - pass - else: - cmd += ": %s" % linkstr - return cmd - - -LibSymlinksAction = SCons.Action.Action(LibSymlinksActionFunction, LibSymlinksStrFun) - - def createSharedLibBuilder(env): """This is a utility function that creates the SharedLibrary Builder in an Environment if it is not there already. diff --git a/SCons/Tool/applelink.py b/SCons/Tool/applelink.py index 8c081a2..4d7b0e2 100644 --- a/SCons/Tool/applelink.py +++ b/SCons/Tool/applelink.py @@ -39,7 +39,8 @@ import SCons.Util # the -rpath option, so we use the "link" tool instead of "gnulink". from . import link -from SCons.Tool import ShLibSonameGenerator +from .linkCommon import ShLibSonameGenerator + class AppleLinkInvalidCurrentVersionException(Exception): pass diff --git a/SCons/Tool/cyglink.py b/SCons/Tool/cyglink.py index fbb6d24..074945f 100644 --- a/SCons/Tool/cyglink.py +++ b/SCons/Tool/cyglink.py @@ -12,19 +12,25 @@ import re import os import SCons.Action +from SCons.Tool.linkCommon import ImpLibSymlinkGenerator, StringizeLibSymlinks, EmitLibSymlinks, ImpLibPrefixGenerator, \ + ImpLibSuffixGenerator, ImpLibNameGenerator import SCons.Util import SCons.Tool -#MAYBE: from . import gnulink from . import gnulink from . import link + def _lib_generator(target, source, env, for_signature, **kw): - try: cmd = kw['cmd'] - except KeyError: cmd = SCons.Util.CLVar(['$SHLINK']) + try: + cmd = kw['cmd'] + except KeyError: + cmd = SCons.Util.CLVar(['$SHLINK']) - try: vp = kw['varprefix'] - except KeyError: vp = 'SHLIB' + try: + vp = kw['varprefix'] + except KeyError: + vp = 'SHLIB' dll = env.FindIxes(target, '%sPREFIX' % vp, '%sSUFFIX' % vp) if dll: cmd.extend(['-o', dll]) @@ -34,12 +40,12 @@ def _lib_generator(target, source, env, for_signature, **kw): implib = env.FindIxes(target, 'IMPLIBPREFIX', 'IMPLIBSUFFIX') if implib: cmd.extend([ - '-Wl,--out-implib='+implib.get_string(for_signature), + '-Wl,--out-implib=' + implib.get_string(for_signature), '-Wl,--export-all-symbols', '-Wl,--enable-auto-import', '-Wl,--whole-archive', '$SOURCES', '-Wl,--no-whole-archive', '$_LIBDIRFLAGS', '$_LIBFLAGS' - ]) + ]) else: cmd.extend(['$SOURCES', '$_LIBDIRFLAGS', '$_LIBFLAGS']) @@ -49,12 +55,14 @@ def _lib_generator(target, source, env, for_signature, **kw): def shlib_generator(target, source, env, for_signature): return _lib_generator(target, source, env, for_signature, varprefix='SHLIB', - cmd = SCons.Util.CLVar(['$SHLINK'])) + cmd=SCons.Util.CLVar(['$SHLINK'])) + def ldmod_generator(target, source, env, for_signature): return _lib_generator(target, source, env, for_signature, varprefix='LDMODULE', - cmd = SCons.Util.CLVar(['$LDMODULE'])) + cmd=SCons.Util.CLVar(['$LDMODULE'])) + def _lib_emitter(target, source, env, **kw): Verbose = False @@ -62,11 +70,15 @@ def _lib_emitter(target, source, env, **kw): if Verbose: print("_lib_emitter: target[0]=%r" % target[0].get_path()) - try: vp = kw['varprefix'] - except KeyError: vp = 'SHLIB' + try: + vp = kw['varprefix'] + except KeyError: + vp = 'SHLIB' - try: libtype = kw['libtype'] - except KeyError: libtype = 'ShLib' + try: + libtype = kw['libtype'] + except KeyError: + libtype = 'ShLib' dll = env.FindIxes(target, '%sPREFIX' % vp, '%sSUFFIX' % vp) no_import_lib = env.get('no_import_lib', 0) @@ -75,12 +87,13 @@ def _lib_emitter(target, source, env, **kw): print("_lib_emitter: dll=%r" % dll.get_path()) if not dll or len(target) > 1: - raise SCons.Errors.UserError("A shared library should have exactly one target with the suffix: %s" % env.subst("$%sSUFFIX" % vp)) + raise SCons.Errors.UserError( + "A shared library should have exactly one target with the suffix: %s" % env.subst("$%sSUFFIX" % vp)) # Remove any "lib" after the prefix pre = env.subst('$%sPREFIX' % vp) - if dll.name[len(pre):len(pre)+3] == 'lib': - dll.name = pre + dll.name[len(pre)+3:] + if dll.name[len(pre):len(pre) + 3] == 'lib': + dll.name = pre + dll.name[len(pre) + 3:] if Verbose: print("_lib_emitter: dll.name=%r" % dll.name) @@ -107,23 +120,26 @@ def _lib_emitter(target, source, env, **kw): implib_target.attributes.shared = 1 target.append(implib_target) - symlinks = SCons.Tool.ImpLibSymlinkGenerator(env, implib_target, - implib_libtype=libtype, - generator_libtype=libtype+'ImpLib') + symlinks = ImpLibSymlinkGenerator(env, implib_target, + implib_libtype=libtype, + generator_libtype=libtype + 'ImpLib') if Verbose: - print("_lib_emitter: implib symlinks=%r" % SCons.Tool.StringizeLibSymlinks(symlinks)) + print("_lib_emitter: implib symlinks=%r" % StringizeLibSymlinks(symlinks)) if symlinks: - SCons.Tool.EmitLibSymlinks(env, symlinks, implib_target, clean_targets = target[0]) + EmitLibSymlinks(env, symlinks, implib_target, clean_targets=target[0]) implib_target.attributes.shliblinks = symlinks return (target, source) + def shlib_emitter(target, source, env): return _lib_emitter(target, source, env, varprefix='SHLIB', libtype='ShLib') + def ldmod_emitter(target, source, env): return _lib_emitter(target, source, env, varprefix='LDMODULE', libtype='LdMod') + def _versioned_lib_suffix(env, suffix, version): """Generate versioned shared library suffix from a unversioned one. If suffix='.dll', and version='0.1.2', then it returns '-0-1-2.dll'""" @@ -138,12 +154,14 @@ def _versioned_lib_suffix(env, suffix, version): print("_versioned_lib_suffix: return suffix= ", suffix) return suffix + def _versioned_implib_name(env, libnode, version, prefix, suffix, **kw): return link._versioned_lib_name(env, libnode, version, prefix, suffix, - SCons.Tool.ImpLibPrefixGenerator, - SCons.Tool.ImpLibSuffixGenerator, + ImpLibPrefixGenerator, + ImpLibSuffixGenerator, implib_libtype=kw['libtype']) + def _versioned_implib_symlinks(env, libnode, version, prefix, suffix, **kw): """Generate link names that should be created for a versioned shared library. Returns a list in the form [ (link, linktarget), ... ] @@ -154,17 +172,18 @@ def _versioned_implib_symlinks(env, libnode, version, prefix, suffix, **kw): print("_versioned_implib_symlinks: libnode=%r" % libnode.get_path()) print("_versioned_implib_symlinks: version=%r" % version) - try: libtype = kw['libtype'] - except KeyError: libtype = 'ShLib' - + try: + libtype = kw['libtype'] + except KeyError: + libtype = 'ShLib' linkdir = os.path.dirname(libnode.get_path()) if Verbose: print("_versioned_implib_symlinks: linkdir=%r" % linkdir) - name = SCons.Tool.ImpLibNameGenerator(env, libnode, - implib_libtype=libtype, - generator_libtype=libtype+'ImpLib') + name = ImpLibNameGenerator(env, libnode, + implib_libtype=libtype, + generator_libtype=libtype + 'ImpLib') if Verbose: print("_versioned_implib_symlinks: name=%r" % name) @@ -174,59 +193,65 @@ def _versioned_implib_symlinks(env, libnode, version, prefix, suffix, **kw): symlinks = [(link0, libnode)] if Verbose: - print("_versioned_implib_symlinks: return symlinks=%r" % SCons.Tool.StringizeLibSymlinks(symlinks)) + print("_versioned_implib_symlinks: return symlinks=%r" % linkStringizeLibSymlinks(symlinks)) return symlinks + shlib_action = SCons.Action.Action(shlib_generator, generator=1) ldmod_action = SCons.Action.Action(ldmod_generator, generator=1) + def generate(env): """Add Builders and construction variables for cyglink to an Environment.""" gnulink.generate(env) - env['LINKFLAGS'] = SCons.Util.CLVar('-Wl,-no-undefined') + env['LINKFLAGS'] = SCons.Util.CLVar('-Wl,-no-undefined') env['SHLINKCOM'] = shlib_action env['LDMODULECOM'] = ldmod_action - env.Append(SHLIBEMITTER = [shlib_emitter]) - env.Append(LDMODULEEMITTER = [ldmod_emitter]) + env.Append(SHLIBEMITTER=[shlib_emitter]) + env.Append(LDMODULEEMITTER=[ldmod_emitter]) - env['SHLIBPREFIX'] = 'cyg' - env['SHLIBSUFFIX'] = '.dll' + env['SHLIBPREFIX'] = 'cyg' + env['SHLIBSUFFIX'] = '.dll' - env['IMPLIBPREFIX'] = 'lib' - env['IMPLIBSUFFIX'] = '.dll.a' + env['IMPLIBPREFIX'] = 'lib' + env['IMPLIBSUFFIX'] = '.dll.a' # Variables used by versioned shared libraries - env['_SHLIBVERSIONFLAGS'] = '$SHLIBVERSIONFLAGS' - env['_LDMODULEVERSIONFLAGS'] = '$LDMODULEVERSIONFLAGS' + env['_SHLIBVERSIONFLAGS'] = '$SHLIBVERSIONFLAGS' + env['_LDMODULEVERSIONFLAGS'] = '$LDMODULEVERSIONFLAGS' # SHLIBVERSIONFLAGS and LDMODULEVERSIONFLAGS are same as in gnulink... # LINKCALLBACKS are NOT inherited from gnulink env['LINKCALLBACKS'] = { - 'VersionedShLibSuffix' : _versioned_lib_suffix, - 'VersionedLdModSuffix' : _versioned_lib_suffix, - 'VersionedImpLibSuffix' : _versioned_lib_suffix, - 'VersionedShLibName' : link._versioned_shlib_name, - 'VersionedLdModName' : link._versioned_ldmod_name, - 'VersionedShLibImpLibName' : lambda *args: _versioned_implib_name(*args, libtype='ShLib'), - 'VersionedLdModImpLibName' : lambda *args: _versioned_implib_name(*args, libtype='LdMod'), - 'VersionedShLibImpLibSymlinks' : lambda *args: _versioned_implib_symlinks(*args, libtype='ShLib'), - 'VersionedLdModImpLibSymlinks' : lambda *args: _versioned_implib_symlinks(*args, libtype='LdMod'), + 'VersionedShLibSuffix': _versioned_lib_suffix, + 'VersionedLdModSuffix': _versioned_lib_suffix, + 'VersionedImpLibSuffix': _versioned_lib_suffix, + 'VersionedShLibName': link._versioned_shlib_name, + 'VersionedLdModName': link._versioned_ldmod_name, + 'VersionedShLibImpLibName': lambda *args: _versioned_implib_name(*args, libtype='ShLib'), + 'VersionedLdModImpLibName': lambda *args: _versioned_implib_name(*args, libtype='LdMod'), + 'VersionedShLibImpLibSymlinks': lambda *args: _versioned_implib_symlinks(*args, libtype='ShLib'), + 'VersionedLdModImpLibSymlinks': lambda *args: _versioned_implib_symlinks(*args, libtype='LdMod'), } # these variables were set by gnulink but are not used in cyglink - try: del env['_SHLIBSONAME'] - except KeyError: pass - try: del env['_LDMODULESONAME'] - except KeyError: pass + try: + del env['_SHLIBSONAME'] + except KeyError: + pass + try: + del env['_LDMODULESONAME'] + except KeyError: + pass + def exists(env): return gnulink.exists(env) - # Local Variables: # tab-width:4 # indent-tabs-mode:nil diff --git a/SCons/Tool/dmd.py b/SCons/Tool/dmd.py index ba12131..5970246 100644 --- a/SCons/Tool/dmd.py +++ b/SCons/Tool/dmd.py @@ -50,8 +50,6 @@ LIBS """ # -# __COPYRIGHT__ -# # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including @@ -72,11 +70,6 @@ LIBS # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - -import os -import subprocess - import SCons.Action import SCons.Builder import SCons.Defaults @@ -84,6 +77,7 @@ import SCons.Scanner.D import SCons.Tool import SCons.Tool.DCommon as DCommon +from SCons.Tool.linkCommon import ShLibSonameGenerator def generate(env): @@ -128,7 +122,8 @@ def generate(env): env['SHDLINK'] = '$DC' env['SHDLINKFLAGS'] = SCons.Util.CLVar('$DLINKFLAGS -shared -defaultlib=libphobos2.so') - env['SHDLINKCOM'] = '$DLINK -of$TARGET $SHDLINKFLAGS $__SHDLIBVERSIONFLAGS $__DRPATH $SOURCES $_DLIBDIRFLAGS $_DLIBFLAGS' + env[ + 'SHDLINKCOM'] = '$DLINK -of$TARGET $SHDLINKFLAGS $__SHDLIBVERSIONFLAGS $__DRPATH $SOURCES $_DLIBDIRFLAGS $_DLIBFLAGS' env['DLIBLINKPREFIX'] = '' if env['PLATFORM'] == 'win32' else '-L-l' env['DLIBLINKSUFFIX'] = '.lib' if env['PLATFORM'] == 'win32' else '' @@ -139,7 +134,8 @@ def generate(env): env['_DLIBDIRFLAGS'] = '${_concat(DLIBDIRPREFIX, LIBPATH, DLIBDIRSUFFIX, __env__, RDirs, TARGET, SOURCE)}' env['DLIB'] = 'lib' if env['PLATFORM'] == 'win32' else 'ar cr' - env['DLIBCOM'] = '$DLIB $_DLIBFLAGS {0}$TARGET $SOURCES $_DLIBFLAGS'.format('-c ' if env['PLATFORM'] == 'win32' else '') + env['DLIBCOM'] = '$DLIB $_DLIBFLAGS {0}$TARGET $SOURCES $_DLIBFLAGS'.format( + '-c ' if env['PLATFORM'] == 'win32' else '') # env['_DLIBFLAGS'] = '${_concat(DLIBFLAGPREFIX, DLIBFLAGS, DLIBFLAGSUFFIX, __env__)}' @@ -157,7 +153,7 @@ def generate(env): env['_SHDLIBSONAME'] = '${DShLibSonameGenerator(__env__,TARGET)}' # NOTE: this is a quick hack, the soname will only work if there is # c/c++ linker loaded which provides callback for the ShLibSonameGenerator - env['DShLibSonameGenerator'] = SCons.Tool.ShLibSonameGenerator + env['DShLibSonameGenerator'] = ShLibSonameGenerator # NOTE: this is only for further reference, currently $SHDLIBVERSION does # not work, the user must use $SHLIBVERSION env['SHDLIBVERSION'] = '$SHLIBVERSION' @@ -172,7 +168,6 @@ def generate(env): def exists(env): return env.Detect(['dmd', 'ldmd2', 'gdmd']) - # Local Variables: # tab-width:4 # indent-tabs-mode:nil diff --git a/SCons/Tool/gdc.py b/SCons/Tool/gdc.py index ecf4f3a..12c327e 100644 --- a/SCons/Tool/gdc.py +++ b/SCons/Tool/gdc.py @@ -53,6 +53,7 @@ import SCons.Defaults import SCons.Tool import SCons.Tool.DCommon as DCommon +import SCons.Tool.linkCommon def generate(env): @@ -120,7 +121,7 @@ def generate(env): env['_SHDLIBSONAME'] = '${DShLibSonameGenerator(__env__,TARGET)}' # NOTE: this is a quick hack, the soname will only work if there is # c/c++ linker loaded which provides callback for the ShLibSonameGenerator - env['DShLibSonameGenerator'] = SCons.Tool.ShLibSonameGenerator + env['DShLibSonameGenerator'] = SCons.Tool.linkCommon.ShLibSonameGenerator # NOTE: this is only for further reference, currently $SHDLIBVERSION does # not work, the user must use $SHLIBVERSION env['SHDLIBVERSION'] = '$SHLIBVERSION' diff --git a/SCons/Tool/install.py b/SCons/Tool/install.py index 67c9ec8..e79203e 100644 --- a/SCons/Tool/install.py +++ b/SCons/Tool/install.py @@ -29,13 +29,13 @@ selection method. # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os from shutil import copy2, copymode, copystat import SCons.Action import SCons.Tool +from SCons.Tool.linkCommon import StringizeLibSymlinks, CreateLibSymlinks, EmitLibSymlinks import SCons.Util # @@ -213,9 +213,9 @@ def installShlibLinks(dest, source, env): Verbose = False symlinks = listShlibLinksToInstall(dest, source, env) if Verbose: - print('installShlibLinks: symlinks={!r}'.format(SCons.Tool.StringizeLibSymlinks(symlinks))) + print('installShlibLinks: symlinks={!r}'.format(StringizeLibSymlinks(symlinks))) if symlinks: - SCons.Tool.CreateLibSymlinks(env, symlinks) + CreateLibSymlinks(env, symlinks) return def installFunc(target, source, env): @@ -292,7 +292,7 @@ def add_versioned_targets_to_INSTALLED_FILES(target, source, env): print("add_versioned_targets_to_INSTALLED_FILES: target={!r}".format(list(map(str, target)))) symlinks = listShlibLinksToInstall(target[0], source, env) if symlinks: - SCons.Tool.EmitLibSymlinks(env, symlinks, target[0]) + EmitLibSymlinks(env, symlinks, target[0]) _UNIQUE_INSTALLED_FILES = None return (target, source) diff --git a/SCons/Tool/ldc.py b/SCons/Tool/ldc.py index f915569..d893841 100644 --- a/SCons/Tool/ldc.py +++ b/SCons/Tool/ldc.py @@ -24,8 +24,6 @@ Lib tool variables: """ # -# __COPYRIGHT__ -# # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including @@ -46,11 +44,6 @@ Lib tool variables: # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - -import os -import subprocess - import SCons.Action import SCons.Builder import SCons.Defaults @@ -58,6 +51,7 @@ import SCons.Scanner.D import SCons.Tool import SCons.Tool.DCommon as DCommon +from SCons.Tool.linkCommon import ShLibSonameGenerator def generate(env): @@ -133,7 +127,7 @@ def generate(env): env['_SHDLIBSONAME'] = '${DShLibSonameGenerator(__env__,TARGET)}' # NOTE: this is a quick hack, the soname will only work if there is # c/c++ linker loaded which provides callback for the ShLibSonameGenerator - env['DShLibSonameGenerator'] = SCons.Tool.ShLibSonameGenerator + env['DShLibSonameGenerator'] = ShLibSonameGenerator # NOTE: this is only for further reference, currently $SHDLIBVERSION does # not work, the user must use $SHLIBVERSION env['SHDLIBVERSION'] = '$SHLIBVERSION' diff --git a/SCons/Tool/link.py b/SCons/Tool/link.py index a3567aa..de377ac 100644 --- a/SCons/Tool/link.py +++ b/SCons/Tool/link.py @@ -9,8 +9,6 @@ selection method. """ # -# __COPYRIGHT__ -# # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including @@ -30,7 +28,6 @@ selection method. # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import sys import re @@ -46,7 +43,9 @@ from SCons.Tool.DCommon import isD from SCons.Tool.cxx import iscplusplus -from SCons.Tool import ShLibSonameGenerator +from SCons.Tool.linkCommon import StringizeLibSymlinks, ShLibSonameGenerator, EmitLibSymlinks, ShLibSymlinkGenerator, \ + LdModSymlinkGenerator, ShLibPrefixGenerator, ShLibSuffixGenerator, LdModPrefixGenerator, LdModSuffixGenerator, \ + LdModSonameGenerator issued_mixed_link_warning = False @@ -97,17 +96,17 @@ def _lib_emitter(target, source, env, **kw): print("_lib_emitter: symlinks={!r}".format(symlinks)) if symlinks: - SCons.Tool.EmitLibSymlinks(env, symlinks, target[0]) + EmitLibSymlinks(env, symlinks, target[0]) target[0].attributes.shliblinks = symlinks return (target, source) def shlib_emitter(target, source, env): - return _lib_emitter(target, source, env, symlink_generator=SCons.Tool.ShLibSymlinkGenerator) + return _lib_emitter(target, source, env, symlink_generator=ShLibSymlinkGenerator) def ldmod_emitter(target, source, env): - return _lib_emitter(target, source, env, symlink_generator=SCons.Tool.LdModSymlinkGenerator) + return _lib_emitter(target, source, env, symlink_generator=LdModSymlinkGenerator) # This is generic enough to be included here... @@ -142,14 +141,14 @@ def _versioned_lib_name(env, libnode, version, prefix, suffix, prefix_generator, def _versioned_shlib_name(env, libnode, version, prefix, suffix, **kw): - prefix_generator = SCons.Tool.ShLibPrefixGenerator - suffix_generator = SCons.Tool.ShLibSuffixGenerator + prefix_generator = ShLibPrefixGenerator + suffix_generator = ShLibSuffixGenerator return _versioned_lib_name(env, libnode, version, prefix, suffix, prefix_generator, suffix_generator, **kw) def _versioned_ldmod_name(env, libnode, version, prefix, suffix, **kw): - prefix_generator = SCons.Tool.LdModPrefixGenerator - suffix_generator = SCons.Tool.LdModSuffixGenerator + prefix_generator = LdModPrefixGenerator + suffix_generator = LdModSuffixGenerator return _versioned_lib_name(env, libnode, version, prefix, suffix, prefix_generator, suffix_generator, **kw) @@ -236,7 +235,8 @@ def _versioned_lib_symlinks(env, libnode, version, prefix, suffix, name_func, so symlinks = [(link0, libnode), (link1, libnode)] if Verbose: - print("_versioned_lib_symlinks: return symlinks={!r}".format(SCons.Tool.StringizeLibSymlinks(symlinks))) + print("_versioned_lib_symlinks: return symlinks={!r}".format( + StringizeLibSymlinks(symlinks))) return symlinks @@ -301,8 +301,8 @@ def _setup_versioned_lib_variables(env, **kw): env['_LDMODULEVERSIONFLAGS'] = '$LDMODULEVERSIONFLAGS -Wl,-soname=$_LDMODULESONAME' env['_SHLIBSONAME'] = '${ShLibSonameGenerator(__env__,TARGET)}' env['_LDMODULESONAME'] = '${LdModSonameGenerator(__env__,TARGET)}' - env['ShLibSonameGenerator'] = SCons.Tool.ShLibSonameGenerator - env['LdModSonameGenerator'] = SCons.Tool.LdModSonameGenerator + env['ShLibSonameGenerator'] = ShLibSonameGenerator + env['LdModSonameGenerator'] = LdModSonameGenerator else: env['_SHLIBVERSIONFLAGS'] = '$SHLIBVERSIONFLAGS' env['_LDMODULEVERSIONFLAGS'] = '$LDMODULEVERSIONFLAGS' diff --git a/SCons/Tool/linkCommon/__init__.py b/SCons/Tool/linkCommon/__init__.py new file mode 100644 index 0000000..f11aa14 --- /dev/null +++ b/SCons/Tool/linkCommon/__init__.py @@ -0,0 +1,503 @@ +"""SCons.Tool.linkCommon + +Common link/shared library logic +""" + +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +import os +from typing import Callable +from SCons.Util import is_List + + +def _call_linker_cb(env, callback, args, result=None): + """Returns the result of env['LINKCALLBACKS'][callback](*args) + if env['LINKCALLBACKS'] is a dictionary and env['LINKCALLBACKS'][callback] + is callable. If these conditions are not met, return the value provided as + the *result* argument. This function is mainly used for generating library + info such as versioned suffixes, symlink maps, sonames etc. by delegating + the core job to callbacks configured by current linker tool""" + + Verbose = False + + if Verbose: + print('_call_linker_cb: args=%r' % args) + print('_call_linker_cb: callback=%r' % callback) + + try: + cbfun = env['LINKCALLBACKS'][callback] + except (KeyError, TypeError): + if Verbose: + print('_call_linker_cb: env["LINKCALLBACKS"][%r] not found or can not be used' % callback) + pass + else: + if Verbose: + print('_call_linker_cb: env["LINKCALLBACKS"][%r] found' % callback) + print('_call_linker_cb: env["LINKCALLBACKS"][%r]=%r' % (callback, cbfun)) + if isinstance(cbfun, Callable): + if Verbose: + print('_call_linker_cb: env["LINKCALLBACKS"][%r] is callable' % callback) + result = cbfun(env, *args) + return result + + +class _ShLibInfoSupport: + @property + def libtype(self): + return 'ShLib' + + def get_lib_prefix(self, env, *args, **kw): + return _call_env_subst(env, '$SHLIBPREFIX', *args, **kw) + + def get_lib_suffix(self, env, *args, **kw): + return _call_env_subst(env, '$SHLIBSUFFIX', *args, **kw) + + def get_lib_version(self, env, *args, **kw): + return _call_env_subst(env, '$SHLIBVERSION', *args, **kw) + + def get_lib_noversionsymlinks(self, env, *args, **kw): + return _call_env_subst(env, '$SHLIBNOVERSIONSYMLINKS', *args, **kw) + + +class _LdModInfoSupport: + @property + def libtype(self): + return 'LdMod' + + def get_lib_prefix(self, env, *args, **kw): + return _call_env_subst(env, '$LDMODULEPREFIX', *args, **kw) + + def get_lib_suffix(self, env, *args, **kw): + return _call_env_subst(env, '$LDMODULESUFFIX', *args, **kw) + + def get_lib_version(self, env, *args, **kw): + return _call_env_subst(env, '$LDMODULEVERSION', *args, **kw) + + def get_lib_noversionsymlinks(self, env, *args, **kw): + return _call_env_subst(env, '$LDMODULENOVERSIONSYMLINKS', *args, **kw) + + +class _ImpLibInfoSupport: + @property + def libtype(self): + return 'ImpLib' + + def get_lib_prefix(self, env, *args, **kw): + return _call_env_subst(env, '$IMPLIBPREFIX', *args, **kw) + + def get_lib_suffix(self, env, *args, **kw): + return _call_env_subst(env, '$IMPLIBSUFFIX', *args, **kw) + + def get_lib_version(self, env, *args, **kw): + version = _call_env_subst(env, '$IMPLIBVERSION', *args, **kw) + if not version: + try: + lt = kw['implib_libtype'] + except KeyError: + pass + else: + if lt == 'ShLib': + version = _call_env_subst(env, '$SHLIBVERSION', *args, **kw) + elif lt == 'LdMod': + version = _call_env_subst(env, '$LDMODULEVERSION', *args, **kw) + return version + + def get_lib_noversionsymlinks(self, env, *args, **kw): + disable = None + try: + env['IMPLIBNOVERSIONSYMLINKS'] + except KeyError: + try: + lt = kw['implib_libtype'] + except KeyError: + pass + else: + if lt == 'ShLib': + disable = _call_env_subst(env, '$SHLIBNOVERSIONSYMLINKS', *args, **kw) + elif lt == 'LdMod': + disable = _call_env_subst(env, '$LDMODULENOVERSIONSYMLINKS', *args, **kw) + else: + disable = _call_env_subst(env, '$IMPLIBNOVERSIONSYMLINKS', *args, **kw) + return disable + + +class _LibInfoGeneratorBase: + """Generator base class for library-related info such as suffixes for + versioned libraries, symlink maps, sonames etc. It handles commonities + of SharedLibrary and LoadableModule + """ + _support_classes = {'ShLib': _ShLibInfoSupport, + 'LdMod': _LdModInfoSupport, + 'ImpLib': _ImpLibInfoSupport} + + def __init__(self, libtype, infoname): + self.libtype = libtype + self.infoname = infoname + + @property + def libtype(self): + return self._support.libtype + + @libtype.setter + def libtype(self, libtype): + try: + support_class = self._support_classes[libtype] + except KeyError: + raise ValueError('unsupported libtype %r' % libtype) + self._support = support_class() + + def get_lib_prefix(self, env, *args, **kw): + return self._support.get_lib_prefix(env, *args, **kw) + + def get_lib_suffix(self, env, *args, **kw): + return self._support.get_lib_suffix(env, *args, **kw) + + def get_lib_version(self, env, *args, **kw): + return self._support.get_lib_version(env, *args, **kw) + + def get_lib_noversionsymlinks(self, env, *args, **kw): + return self._support.get_lib_noversionsymlinks(env, *args, **kw) + + def get_versioned_lib_info_generator(self, **kw): + """ + Returns name of generator linker callback that will be used to generate + our info for a versioned library. For example, if our libtype is 'ShLib' + and infoname is 'Prefix', it would return 'VersionedShLibPrefix'. + """ + try: + libtype = kw['generator_libtype'] + except KeyError: + libtype = self.libtype + return 'Versioned%s%s' % (libtype, self.infoname) + + def generate_versioned_lib_info(self, env, args, result=None, **kw): + callback = self.get_versioned_lib_info_generator(**kw) + return _call_linker_cb(env, callback, args, result) + + +class _LibPrefixGenerator(_LibInfoGeneratorBase): + """Library prefix generator, used as target_prefix in SharedLibrary and + LoadableModule builders""" + + def __init__(self, libtype): + super(_LibPrefixGenerator, self).__init__(libtype, 'Prefix') + + def __call__(self, env, sources=None, **kw): + Verbose = False + + if sources and 'source' not in kw: + kw2 = kw.copy() + kw2['source'] = sources + else: + kw2 = kw + + prefix = self.get_lib_prefix(env, **kw2) + if Verbose: + print("_LibPrefixGenerator: input prefix=%r" % prefix) + + version = self.get_lib_version(env, **kw2) + if Verbose: + print("_LibPrefixGenerator: version=%r" % version) + + if version: + prefix = self.generate_versioned_lib_info(env, [prefix, version], prefix, **kw2) + + if Verbose: + print("_LibPrefixGenerator: return prefix=%r" % prefix) + return prefix + + +ShLibPrefixGenerator = _LibPrefixGenerator('ShLib') +LdModPrefixGenerator = _LibPrefixGenerator('LdMod') +ImpLibPrefixGenerator = _LibPrefixGenerator('ImpLib') + + +class _LibSuffixGenerator(_LibInfoGeneratorBase): + """Library suffix generator, used as target_suffix in SharedLibrary and + LoadableModule builders""" + + def __init__(self, libtype): + super(_LibSuffixGenerator, self).__init__(libtype, 'Suffix') + + def __call__(self, env, sources=None, **kw): + Verbose = False + + if sources and 'source' not in kw: + kw2 = kw.copy() + kw2['source'] = sources + else: + kw2 = kw + + suffix = self.get_lib_suffix(env, **kw2) + if Verbose: + print("_LibSuffixGenerator: input suffix=%r" % suffix) + + version = self.get_lib_version(env, **kw2) + if Verbose: + print("_LibSuffixGenerator: version=%r" % version) + + if version: + suffix = self.generate_versioned_lib_info(env, [suffix, version], suffix, **kw2) + + if Verbose: + print("_LibSuffixGenerator: return suffix=%r" % suffix) + return suffix + + +ShLibSuffixGenerator = _LibSuffixGenerator('ShLib') +LdModSuffixGenerator = _LibSuffixGenerator('LdMod') +ImpLibSuffixGenerator = _LibSuffixGenerator('ImpLib') + + +class _LibSymlinkGenerator(_LibInfoGeneratorBase): + """Library symlink map generator. It generates a list of symlinks that + should be created by SharedLibrary or LoadableModule builders""" + + def __init__(self, libtype): + super(_LibSymlinkGenerator, self).__init__(libtype, 'Symlinks') + + def __call__(self, env, libnode, **kw): + Verbose = False + + if libnode and 'target' not in kw: + kw2 = kw.copy() + kw2['target'] = libnode + else: + kw2 = kw + + if Verbose: + print("_LibSymLinkGenerator: libnode=%r" % libnode.get_path()) + + symlinks = None + + version = self.get_lib_version(env, **kw2) + disable = self.get_lib_noversionsymlinks(env, **kw2) + if Verbose: + print('_LibSymlinkGenerator: version=%r' % version) + print('_LibSymlinkGenerator: disable=%r' % disable) + + if version and not disable: + prefix = self.get_lib_prefix(env, **kw2) + suffix = self.get_lib_suffix(env, **kw2) + symlinks = self.generate_versioned_lib_info(env, [libnode, version, prefix, suffix], **kw2) + + if Verbose: + print('_LibSymlinkGenerator: return symlinks=%r' % StringizeLibSymlinks(symlinks)) + return symlinks + + +ShLibSymlinkGenerator = _LibSymlinkGenerator('ShLib') +LdModSymlinkGenerator = _LibSymlinkGenerator('LdMod') +ImpLibSymlinkGenerator = _LibSymlinkGenerator('ImpLib') + + +class _LibNameGenerator(_LibInfoGeneratorBase): + """Generates "unmangled" library name from a library file node. + + Generally, it's thought to revert modifications done by prefix/suffix + generators (_LibPrefixGenerator/_LibSuffixGenerator) used by a library + builder. For example, on gnulink the suffix generator used by SharedLibrary + builder appends $SHLIBVERSION to $SHLIBSUFFIX producing node name which + ends with "$SHLIBSUFFIX.$SHLIBVERSION". Correspondingly, the implementation + of _LibNameGenerator replaces "$SHLIBSUFFIX.$SHLIBVERSION" with + "$SHLIBSUFFIX" in the node's basename. So that, if $SHLIBSUFFIX is ".so", + $SHLIBVERSION is "0.1.2" and the node path is "/foo/bar/libfoo.so.0.1.2", + the _LibNameGenerator shall return "libfoo.so". Other link tools may + implement it's own way of library name unmangling. + """ + + def __init__(self, libtype): + super(_LibNameGenerator, self).__init__(libtype, 'Name') + + def __call__(self, env, libnode, **kw): + """Returns "demangled" library name""" + Verbose = False + + if libnode and 'target' not in kw: + kw2 = kw.copy() + kw2['target'] = libnode + else: + kw2 = kw + + if Verbose: + print("_LibNameGenerator: libnode=%r" % libnode.get_path()) + + version = self.get_lib_version(env, **kw2) + if Verbose: + print('_LibNameGenerator: version=%r' % version) + + name = None + if version: + prefix = self.get_lib_prefix(env, **kw2) + suffix = self.get_lib_suffix(env, **kw2) + name = self.generate_versioned_lib_info(env, [libnode, version, prefix, suffix], **kw2) + + if not name: + name = os.path.basename(libnode.get_path()) + + if Verbose: + print('_LibNameGenerator: return name=%r' % name) + + return name + + +ShLibNameGenerator = _LibNameGenerator('ShLib') +LdModNameGenerator = _LibNameGenerator('LdMod') +ImpLibNameGenerator = _LibNameGenerator('ImpLib') + + +class _LibSonameGenerator(_LibInfoGeneratorBase): + """Library soname generator. Returns library soname (e.g. libfoo.so.0) for + a given node (e.g. /foo/bar/libfoo.so.0.1.2)""" + + def __init__(self, libtype): + super(_LibSonameGenerator, self).__init__(libtype, 'Soname') + + def __call__(self, env, libnode, **kw): + """Returns a SONAME based on a shared library's node path""" + Verbose = False + + if libnode and 'target' not in kw: + kw2 = kw.copy() + kw2['target'] = libnode + else: + kw2 = kw + + if Verbose: + print("_LibSonameGenerator: libnode=%r" % libnode.get_path()) + + soname = _call_env_subst(env, '$SONAME', **kw2) + if not soname: + version = self.get_lib_version(env, **kw2) + if Verbose: + print("_LibSonameGenerator: version=%r" % version) + if version: + prefix = self.get_lib_prefix(env, **kw2) + suffix = self.get_lib_suffix(env, **kw2) + soname = self.generate_versioned_lib_info(env, [libnode, version, prefix, suffix], **kw2) + + if not soname: + # fallback to library name (as returned by appropriate _LibNameGenerator) + soname = _LibNameGenerator(self.libtype)(env, libnode) + if Verbose: + print("_LibSonameGenerator: FALLBACK: soname=%r" % soname) + + if Verbose: + print("_LibSonameGenerator: return soname=%r" % soname) + + return soname + + +ShLibSonameGenerator = _LibSonameGenerator('ShLib') +LdModSonameGenerator = _LibSonameGenerator('LdMod') + + +def StringizeLibSymlinks(symlinks): + """Converts list with pairs of nodes to list with pairs of node paths + (strings). Used mainly for debugging.""" + if is_List(symlinks): + try: + return [(k.get_path(), v.get_path()) for k, v in symlinks] + except (TypeError, ValueError): + return symlinks + else: + return symlinks + + +def EmitLibSymlinks(env, symlinks, libnode, **kw): + """Used by emitters to handle (shared/versioned) library symlinks""" + Verbose = False + + # nodes involved in process... all symlinks + library + nodes = list(set([x for x, y in symlinks] + [libnode])) + + clean_targets = kw.get('clean_targets', []) + if not is_List(clean_targets): + clean_targets = [clean_targets] + + for link, linktgt in symlinks: + env.SideEffect(link, linktgt) + if Verbose: + print("EmitLibSymlinks: SideEffect(%r,%r)" % (link.get_path(), linktgt.get_path())) + clean_list = [x for x in nodes if x != linktgt] + env.Clean(list(set([linktgt] + clean_targets)), clean_list) + if Verbose: + print("EmitLibSymlinks: Clean(%r,%r)" % (linktgt.get_path(), [x.get_path() for x in clean_list])) + + +def CreateLibSymlinks(env, symlinks): + """Physically creates symlinks. The symlinks argument must be a list in + form [ (link, linktarget), ... ], where link and linktarget are SCons + nodes. + """ + + Verbose = False + for link, linktgt in symlinks: + linktgt = link.get_dir().rel_path(linktgt) + link = link.get_path() + if Verbose: + print("CreateLibSymlinks: preparing to add symlink %r -> %r" % (link, linktgt)) + # Delete the (previously created) symlink if exists. Let only symlinks + # to be deleted to prevent accidental deletion of source files... + if env.fs.islink(link): + env.fs.unlink(link) + if Verbose: + print("CreateLibSymlinks: removed old symlink %r" % link) + # If a file or directory exists with the same name as link, an OSError + # will be thrown, which should be enough, I think. + env.fs.symlink(linktgt, link) + if Verbose: + print("CreateLibSymlinks: add symlink %r -> %r" % (link, linktgt)) + return 0 + + +def LibSymlinksActionFunction(target, source, env): + for tgt in target: + symlinks = getattr(getattr(tgt, 'attributes', None), 'shliblinks', None) + if symlinks: + CreateLibSymlinks(env, symlinks) + return 0 + + +def LibSymlinksStrFun(target, source, env, *args): + cmd = None + for tgt in target: + symlinks = getattr(getattr(tgt, 'attributes', None), 'shliblinks', None) + if symlinks: + if cmd is None: cmd = "" + if cmd: cmd += "\n" + cmd += "Create symlinks for: %r" % tgt.get_path() + try: + linkstr = ', '.join(["%r->%r" % (k, v) for k, v in StringizeLibSymlinks(symlinks)]) + except (KeyError, ValueError): + pass + else: + cmd += ": %s" % linkstr + return cmd + + +def _call_env_subst(env, string, *args, **kw): + kw2 = {} + for k in ('raw', 'target', 'source', 'conv', 'executor'): + try: + kw2[k] = kw[k] + except KeyError: + pass + return env.subst(string, *args, **kw2) \ No newline at end of file -- cgit v0.12 From 52f8072e6e18953132fe21a8c99bbb6c62a8dfb8 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Tue, 25 Aug 2020 22:13:14 -0400 Subject: undo use_soname for this branch --- SCons/Tool/applelink.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/SCons/Tool/applelink.py b/SCons/Tool/applelink.py index 4d7b0e2..aec8b93 100644 --- a/SCons/Tool/applelink.py +++ b/SCons/Tool/applelink.py @@ -45,6 +45,7 @@ from .linkCommon import ShLibSonameGenerator class AppleLinkInvalidCurrentVersionException(Exception): pass + class AppleLinkInvalidCompatibilityVersionException(Exception): pass @@ -71,7 +72,7 @@ def _applelib_versioned_lib_soname(env, libnode, version, prefix, suffix, name_f if Verbose: print("_applelib_versioned_lib_soname: name={!r}".format(name)) major = version.split('.')[0] - (libname,_suffix) = name.split('.') + (libname, _suffix) = name.split('.') # if a desired SONAME was supplied, use that, otherwise create # a default from the major version if env.get('SONAME'): @@ -82,12 +83,15 @@ def _applelib_versioned_lib_soname(env, libnode, version, prefix, suffix, name_f print("_applelib_versioned_lib_soname: soname={!r}".format(soname)) return soname + def _applelib_versioned_shlib_soname(env, libnode, version, prefix, suffix): return _applelib_versioned_lib_soname(env, libnode, version, prefix, suffix, link._versioned_shlib_name) # User programmatically describes how SHLIBVERSION maps to values for compat/current. _applelib_max_version_values = (65535, 255, 255) + + def _applelib_check_valid_version(version_string): """ Check that the version # is valid. @@ -100,17 +104,18 @@ def _applelib_check_valid_version(version_string): """ parts = version_string.split('.') if len(parts) > 3: - return False, "Version string has too many periods [%s]"%version_string + return False, "Version string has too many periods [%s]" % version_string if len(parts) <= 0: - return False, "Version string unspecified [%s]"%version_string + return False, "Version string unspecified [%s]" % version_string for (i, p) in enumerate(parts): try: p_i = int(p) except ValueError: - return False, "Version component %s (from %s) is not a number"%(p, version_string) + return False, "Version component %s (from %s) is not a number" % (p, version_string) if p_i < 0 or p_i > _applelib_max_version_values[i]: - return False, "Version component %s (from %s) is not valid value should be between 0 and %d"%(p, version_string, _applelib_max_version_values[i]) + return False, "Version component %s (from %s) is not valid value should be between 0 and %d" % ( + p, version_string, _applelib_max_version_values[i]) return True, "" @@ -191,9 +196,8 @@ def generate(env): env['SHLINKFLAGS'] = SCons.Util.CLVar('$LINKFLAGS -dynamiclib') env['SHLINKCOM'] = env['SHLINKCOM'] + ' $_FRAMEWORKPATH $_FRAMEWORKS $FRAMEWORKSFLAGS' - # see: http://docstore.mik.ua/orelly/unix3/mac/ch05_04.htm for proper naming - link._setup_versioned_lib_variables(env, tool = 'applelink')#, use_soname = use_soname) + link._setup_versioned_lib_variables(env, tool='applelink') #, use_soname=True) env['LINKCALLBACKS'] = link._versioned_lib_callbacks() env['LINKCALLBACKS']['VersionedShLibSuffix'] = _applelib_versioned_lib_suffix env['LINKCALLBACKS']['VersionedShLibSoname'] = _applelib_versioned_shlib_soname @@ -206,15 +210,14 @@ def generate(env): # override the default for loadable modules, which are different # on OS X than dynamic shared libs. echoing what XCode does for # pre/suffixes: - env['LDMODULEPREFIX'] = '' - env['LDMODULESUFFIX'] = '' + env['LDMODULEPREFIX'] = '' + env['LDMODULESUFFIX'] = '' env['LDMODULEFLAGS'] = SCons.Util.CLVar('$LINKFLAGS -bundle') env['LDMODULECOM'] = '$LDMODULE -o ${TARGET} $LDMODULEFLAGS $SOURCES $_LIBDIRFLAGS $_LIBFLAGS $_FRAMEWORKPATH $_FRAMEWORKS $FRAMEWORKSFLAGS' env['__SHLIBVERSIONFLAGS'] = '${__libversionflags(__env__,"SHLIBVERSION","_SHLIBVERSIONFLAGS")}' - def exists(env): return env['PLATFORM'] == 'darwin' -- cgit v0.12 From 36427e8292c1e111b57bdee8bce513cf28e694cc Mon Sep 17 00:00:00 2001 From: William Deegan Date: Tue, 25 Aug 2020 22:14:39 -0400 Subject: Add CHANGES.txt entry --- CHANGES.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.txt b/CHANGES.txt index a359608..81ba741 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -16,6 +16,7 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER From William Deegan: - Fix yacc tool, not respecting YACC set at time of tool initialization. + - Refactor SCons.Tool to move all common shared and loadable module linking logic to SCons.Tool.linkCommon From Adam Gross: - Fix minor bug affecting SCons.Node.FS.File.get_csig()'s usage of the MD5 chunksize. -- cgit v0.12 From b84a457e407082a555a52dc7ea75328670c78f7a Mon Sep 17 00:00:00 2001 From: William Deegan Date: Tue, 25 Aug 2020 22:24:05 -0400 Subject: Fix test/import to skip linkCommon --- test/import.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/import.py b/test/import.py index 99a40d7..df64a1e 100644 --- a/test/import.py +++ b/test/import.py @@ -1,7 +1,5 @@ #!/usr/bin/env python # -# __COPYRIGHT__ -# # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including @@ -22,8 +20,6 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - """ Verify that we can import and use the contents of Platform and Tool modules directly. @@ -78,6 +74,8 @@ ignore = ('__init__.py', 'MSCommon', # clang common "clangCommon", + # link common logic + "linkCommon", # Sun pkgchk and pkginfo common stuff 'sun_pkg.py', # RPM utilities -- cgit v0.12 From 7f0df2151f891497b818aa406e614015132a4224 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Tue, 25 Aug 2020 22:29:44 -0400 Subject: Fix sider complaint which was real missign symbol issue --- SCons/Tool/cyglink.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SCons/Tool/cyglink.py b/SCons/Tool/cyglink.py index 074945f..08b8a98 100644 --- a/SCons/Tool/cyglink.py +++ b/SCons/Tool/cyglink.py @@ -193,7 +193,7 @@ def _versioned_implib_symlinks(env, libnode, version, prefix, suffix, **kw): symlinks = [(link0, libnode)] if Verbose: - print("_versioned_implib_symlinks: return symlinks=%r" % linkStringizeLibSymlinks(symlinks)) + print("_versioned_implib_symlinks: return symlinks=%r" % StringizeLibSymlinks(symlinks)) return symlinks -- cgit v0.12 From f1bc62b1e0dc5fee57a572a5063a708e40702ee0 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Thu, 27 Aug 2020 12:34:58 -0600 Subject: Fix some MergeFlags issues - signature/return did not match documentation or existing usage. Removed "dict" arg; method now does not return self. - merging --param arguments did not work (ParseFlags issue). partly fixes #3107 - passing a dict to merge where the values are strings failed. fixes #2961 Signed-off-by: Mats Wichmann --- CHANGES.txt | 4 +++ SCons/Environment.py | 81 ++++++++++++++++++++++++++---------------- SCons/EnvironmentTests.py | 89 +++++++++++++++++++++++++++++------------------ 3 files changed, 110 insertions(+), 64 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 6293590..e176125 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -35,6 +35,10 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER OverrideEnvironment. Enable env.setdefault() method, add tests. - Raise an error if an option (not otherwise consumed) is used which looks like an abbreviation of one one added by AddOption. (#3653) + - Fix three issues in MergeFlags: signature/return did not match + documentation or existing usage; merging --param arguments did + not work (issue #3107); passing a dict to merge where the values + are strings failed (issue #2961). From Joachim Kuebart: - Suppress missing SConscript deprecation warning if `must_exist=False` diff --git a/SCons/Environment.py b/SCons/Environment.py index bb57e37..c874bd0 100644 --- a/SCons/Environment.py +++ b/SCons/Environment.py @@ -642,16 +642,18 @@ class SubstitutionEnvironment: else: overrides[key] = SCons.Subst.scons_subst_once(value, self, key) env = OverrideEnvironment(self, overrides) - if merges: env.MergeFlags(merges) + if merges: + env.MergeFlags(merges) return env def ParseFlags(self, *flags): - """ - Parse the set of flags and return a dict with the flags placed - in the appropriate entry. The flags are treated as a typical - set of command-line flags for a GNU-like toolchain and used to - populate the entries in the dict immediately below. If one of - the flag strings begins with a bang (exclamation mark), it is + """Return a dict of parsed flags. + + Parse ``flags`` and return a dict with the flags distributed into + the appropriate construction variable names. The flags are treated + as a typical set of command-line flags for a GNU-like toolchain and + used to populate the entries in the dict immediately below. + If one of the flag strings begins with a bang (exclamation mark), it is assumed to be a command and the rest of the string is executed; the result of that evaluation is then added to the dict. """ @@ -741,6 +743,9 @@ class SubstitutionEnvironment: t = ('-arch', arg) dict['CCFLAGS'].append(t) dict['LINKFLAGS'].append(t) + elif append_next_arg_to == '--param': + t = ('--param', arg) + dict['CCFLAGS'].append(t) else: dict[append_next_arg_to].append(arg) append_next_arg_to = None @@ -792,11 +797,13 @@ class SubstitutionEnvironment: dict['FRAMEWORKPATH'].append(arg[2:]) else: append_next_arg_to = 'FRAMEWORKPATH' - elif arg in ['-mno-cygwin', - '-pthread', - '-openmp', - '-fmerge-all-constants', - '-fopenmp']: + elif arg in [ + '-mno-cygwin', + '-pthread', + '-openmp', + '-fmerge-all-constants', + '-fopenmp', + ]: dict['CCFLAGS'].append(arg) dict['LINKFLAGS'].append(arg) elif arg == '-mwindows': @@ -810,7 +817,16 @@ class SubstitutionEnvironment: elif arg[0] == '+': dict['CCFLAGS'].append(arg) dict['LINKFLAGS'].append(arg) - elif arg in ['-include', '-imacros', '-isysroot', '-isystem', '-iquote', '-idirafter', '-arch']: + elif arg in [ + '-include', + '-imacros', + '-isysroot', + '-isystem', + '-iquote', + '-idirafter', + '-arch', + '--param', + ]: append_next_arg_to = arg else: dict['CCFLAGS'].append(arg) @@ -819,24 +835,30 @@ class SubstitutionEnvironment: do_parse(arg) return dict - def MergeFlags(self, args, unique=1, dict=None): - """ - Merge the dict in args into the construction variables of this - env, or the passed-in dict. If args is not a dict, it is - converted into a dict using ParseFlags. If unique is not set, - the flags are appended rather than merged. - """ + def MergeFlags(self, args, unique=True): + """Merge flags into construction variables. + + Merges the flags from ``args`` into this construction environent. + If ``args`` is not a dict, it is first converted to a dictionary with + flags distributed into appropriate construction variables. + See :meth:`ParseFlags`. + + Args: + args: flags to merge + unique: merge flags rather than appending (default: True) - if dict is None: - dict = self + """ if not SCons.Util.is_Dict(args): args = self.ParseFlags(args) + if not unique: self.Append(**args) - return self + return + for key, value in args.items(): if not value: continue + value = SCons.Util.Split(value) try: orig = self[key] except KeyError: @@ -873,7 +895,6 @@ class SubstitutionEnvironment: if v not in t: t.insert(0, v) self[key] = t - return self def default_decide_source(dependency, target, prev_ni, repo_node=None): @@ -1012,7 +1033,8 @@ class Base(SubstitutionEnvironment): self._dict[key] = val # Finally, apply any flags to be merged in - if parse_flags: self.MergeFlags(parse_flags) + if parse_flags: + self.MergeFlags(parse_flags) ####################################################################### # Utility methods that are primarily for internal use by SCons. @@ -1164,9 +1186,7 @@ class Base(SubstitutionEnvironment): ####################################################################### def Append(self, **kw): - """Append values to existing construction variables - in an Environment. - """ + """Append values to existing construction variables in an Environment.""" kw = copy_non_reserved_keywords(kw) for key, val in kw.items(): # It would be easier on the eyes to write this using @@ -1453,7 +1473,8 @@ class Base(SubstitutionEnvironment): clone.Replace(**new) # Finally, apply any flags to be merged in - if parse_flags: clone.MergeFlags(parse_flags) + if parse_flags: + clone.MergeFlags(parse_flags) if SCons.Debug.track_instances: logInstanceCreation(self, 'Environment.EnvironmentClone') return clone @@ -1623,7 +1644,7 @@ class Base(SubstitutionEnvironment): """ if function is None: def parse_conf(env, cmd, unique=unique): - return env.MergeFlags(cmd, unique) + env.MergeFlags(cmd, unique) function = parse_conf if SCons.Util.is_List(command): command = ' '.join(command) diff --git a/SCons/EnvironmentTests.py b/SCons/EnvironmentTests.py index 7a26a61..d2db508 100644 --- a/SCons/EnvironmentTests.py +++ b/SCons/EnvironmentTests.py @@ -787,33 +787,36 @@ sys.exit(0) d = env.ParseFlags([]) assert d == empty, d - s = "-I/usr/include/fum -I bar -X\n" + \ - '-I"C:\\Program Files\\ASCEND\\include" ' + \ - "-L/usr/fax -L foo -lxxx -l yyy " + \ - '-L"C:\\Program Files\\ASCEND" -lascend ' + \ - "-Wa,-as -Wl,-link " + \ - "-Wl,-rpath=rpath1 " + \ - "-Wl,-R,rpath2 " + \ - "-Wl,-Rrpath3 " + \ - "-Wp,-cpp " + \ - "-std=c99 " + \ - "-std=c++0x " + \ - "-framework Carbon " + \ - "-frameworkdir=fwd1 " + \ - "-Ffwd2 " + \ - "-F fwd3 " + \ - "-dylib_file foo-dylib " + \ - "-pthread " + \ - "-fmerge-all-constants " +\ - "-fopenmp " + \ - "-mno-cygwin -mwindows " + \ - "-arch i386 -isysroot /tmp " + \ - "-iquote /usr/include/foo1 " + \ - "-isystem /usr/include/foo2 " + \ - "-idirafter /usr/include/foo3 " + \ - "-imacros /usr/include/foo4 " + \ - "+DD64 " + \ + s = ( + "-I/usr/include/fum -I bar -X " + '-I"C:\\Program Files\\ASCEND\\include" ' + "-L/usr/fax -L foo -lxxx -l yyy " + '-L"C:\\Program Files\\ASCEND" -lascend ' + "-Wa,-as -Wl,-link " + "-Wl,-rpath=rpath1 " + "-Wl,-R,rpath2 " + "-Wl,-Rrpath3 " + "-Wp,-cpp " + "-std=c99 " + "-std=c++0x " + "-framework Carbon " + "-frameworkdir=fwd1 " + "-Ffwd2 " + "-F fwd3 " + "-dylib_file foo-dylib " + "-pthread " + "-fmerge-all-constants " + "-fopenmp " + "-mno-cygwin -mwindows " + "-arch i386 -isysroot /tmp " + "-iquote /usr/include/foo1 " + "-isystem /usr/include/foo2 " + "-idirafter /usr/include/foo3 " + "-imacros /usr/include/foo4 " + "--param l1-cache-size=32 --param l2-cache-size=6144 " + "+DD64 " "-DFOO -DBAR=value -D BAZ " + ) d = env.ParseFlags(s) @@ -827,6 +830,7 @@ sys.exit(0) ('-isystem', '/usr/include/foo2'), ('-idirafter', '/usr/include/foo3'), ('-imacros', env.fs.File('/usr/include/foo4')), + ('--param', 'l1-cache-size=32'), ('--param', 'l2-cache-size=6144'), '+DD64'], repr(d['CCFLAGS']) assert d['CXXFLAGS'] == ['-std=c++0x'], repr(d['CXXFLAGS']) assert d['CPPDEFINES'] == ['FOO', ['BAR', 'value'], 'BAZ'], d['CPPDEFINES'] @@ -852,24 +856,40 @@ sys.exit(0) def test_MergeFlags(self): - """Test the MergeFlags() method - """ + """Test the MergeFlags() method.""" + env = SubstitutionEnvironment() + # does not set flag if value empty env.MergeFlags('') assert 'CCFLAGS' not in env, env['CCFLAGS'] + # merges value if flag did not exist + env.MergeFlags('-X') + assert env['CCFLAGS'] == ['-X'], env['CCFLAGS'] + + # avoid SubstitutionEnvironment for these, has no .Append method, + # which is needed for unique=False test + env = Environment(CCFLAGS=None) + # merge with existing but empty flag env.MergeFlags('-X') assert env['CCFLAGS'] == ['-X'], env['CCFLAGS'] + # default Unique=True enforces no dupes env.MergeFlags('-X') assert env['CCFLAGS'] == ['-X'], env['CCFLAGS'] + # Unique=False allows dupes + env.MergeFlags('-X', unique=False) + assert env['CCFLAGS'] == ['-X', '-X'], env['CCFLAGS'] - env = SubstitutionEnvironment(CCFLAGS=None) - env.MergeFlags('-Y') - assert env['CCFLAGS'] == ['-Y'], env['CCFLAGS'] + # merge from a dict with list values + env = SubstitutionEnvironment(B='b') + env.MergeFlags({'A': ['aaa'], 'B': ['bb', 'bbb']}) + assert env['A'] == ['aaa'], env['A'] + assert env['B'] == ['b', 'bb', 'bbb'], env['B'] - env = SubstitutionEnvironment() - env.MergeFlags({'A':['aaa'], 'B':['bbb']}) + # issue #2961: merge from a dict with string values + env = SubstitutionEnvironment(B='b') + env.MergeFlags({'A': 'aaa', 'B': 'bb bbb'}) assert env['A'] == ['aaa'], env['A'] - assert env['B'] == ['bbb'], env['B'] + assert env['B'] == ['b', 'bb', 'bbb'], env['B'] # issue #3665: if merging dict which is a compound object # (i.e. value can be lists, etc.), the value object should not @@ -881,6 +901,7 @@ sys.exit(0) pass flags = {'CFLAGS': ['-pipe', '-pthread', '-g']} import copy + saveflags = copy.deepcopy(flags) env.MergeFlags(flags) self.assertEqual(flags, saveflags) -- cgit v0.12 From 2235c5c6ac22abfa5af46007ade31d572fe72e6f Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Fri, 28 Aug 2020 08:40:58 -0600 Subject: Updated docs on Actions for output control Manpage now mentions specifically generating your own Action object in order to control the output string. Calls out using None as the command-output specifier to avoid output: Action(foo, None) and Action(foo, cmdstr=None), as well Userguide got a mention of using the action kwarg for readability and an example showing creating the Action for Command manually to set the output string. Signed-off-by: Mats Wichmann --- doc/generated/examples/builderscommands_ex5_1.xml | 3 ++ doc/man/scons.xml | 57 ++++++++++++-------- doc/user/builders-commands.xml | 66 +++++++++++++++++++++-- doc/user/output.xml | 2 +- 4 files changed, 100 insertions(+), 28 deletions(-) create mode 100644 doc/generated/examples/builderscommands_ex5_1.xml diff --git a/doc/generated/examples/builderscommands_ex5_1.xml b/doc/generated/examples/builderscommands_ex5_1.xml new file mode 100644 index 0000000..3811242 --- /dev/null +++ b/doc/generated/examples/builderscommands_ex5_1.xml @@ -0,0 +1,3 @@ +% scons -Q +Building foo.in + diff --git a/doc/man/scons.xml b/doc/man/scons.xml index fc28e2a..96b5700 100644 --- a/doc/man/scons.xml +++ b/doc/man/scons.xml @@ -5521,18 +5521,21 @@ and emitter functions. function will turn its action keyword argument into an appropriate -internal Action object. +internal Action object, as will +the &f-link-Command; function. You can also explicitly create Action objects for passing to &f-Builder;, or other functions that take actions as arguments, by calling the &f-link-Action; factory function. -This can be used to configure -an Action object more flexibly, -or it may simply be more efficient -than letting each separate Builder object -create a separate Action -when multiple -Builder objects need to do the same thing. +This may more efficient when multiple +Builder objects need to do the same thing +rather than letting each of those Builder objects +create a separate Action object. +It also allows more flexible configuration +of an Action object. For example, to control +the message printed when the action is taken +you need to create the action object using &f-Action;. + The &Action; factory function @@ -5652,23 +5655,22 @@ it was called. The global function form &f-link-Action; the Action object is actually used. -The second argument to &f-Action; -is optional and is used to define the output +The optional second argument to &f-Action; +is used to control the output which is printed when the Action is actually performed. -In the absence of this parameter, -or if it's an empty string, +If this parameter is omitted, +or if the value is an empty string, a default output depending on the type of the action is used. For example, a command-line action will print the executed command. -The argument must be either a Python function or a string: +The following argument types are accepted: + If the output argument is a function, -it must return a string +the function will be called to obtain a string describing the action being executed. -A function may also be specified using the -strfunction -keyword argument. The function +The function must accept these three keyword arguments: @@ -5689,18 +5691,29 @@ more than one target file or source file. If the output argument is a string, -substitution is performed on it before it is printed. -The output string may also be specified using the -cmdstr -keyword argument. +substitution is performed on the string before it is printed. The string typically contains variables, notably $TARGET(S) and $SOURCE(S), or consists of just a single variable, which is optionally defined somewhere else. -SCons itself heavily uses the latter variant. +&SCons; itself heavily uses the latter variant. + + + +If the argument is None, +output is suppressed entirely. + +Instead of using a positional argument, +the cmdstr +keyword argument may be used to specify the output string, +or the strfunction keyword argument +may be used to specify a function to return the output string, +Use cmdstr=None to suppress output. + + Examples: diff --git a/doc/user/builders-commands.xml b/doc/user/builders-commands.xml index 5d378b3..9bbd962 100644 --- a/doc/user/builders-commands.xml +++ b/doc/user/builders-commands.xml @@ -81,7 +81,7 @@ if you only need to execute one specific command to build a single file (or group of files). For these situations, &SCons; supports a - &Command; &Builder; that arranges + &f-link-Command; builder that arranges for a specific action to be executed to build a specific file or files. This looks a lot like the other builders @@ -119,7 +119,7 @@ foo.in This is often more convenient than creating a &Builder; object and adding it to the &cv-link-BUILDERS; variable - of a &consenv; + of a &consenv;. @@ -134,9 +134,11 @@ foo.in env = Environment() + def build(target, source, env): # Whatever it takes to build return None + env.Command('foo.out', 'foo.in', build) @@ -157,8 +159,7 @@ foo.in Note that &cv-link-SOURCE; and &cv-link-TARGET; are expanded - in the source and target as well as of SCons 1.1, - so you can write: + in the source and target as well, so you can write: @@ -168,7 +169,6 @@ env.Command('${SOURCE.basename}.out', 'foo.in', build) - which does the same thing as the previous example, but allows you @@ -176,5 +176,61 @@ env.Command('${SOURCE.basename}.out', 'foo.in', build) + + + It may be helpful to use the action + keyword to specify the action, is this makes things more clear + to the reader: + + + + + +env.Command('${SOURCE.basename}.out', 'foo.in', action=build) + + + + + + The method described in + for controlling + build output works well when used with pre-defined builders which + have pre-defined *COMSTR variables for that purpose, + but that is not the case when calling &f-Command;, + where &SCons; has no specific knowledge of the action ahead of time. + If the action argument to &f-Command is not already an &Action; object, + it will construct one for you with suitable defaults, + which include a message based on the type of action. + However, you can also construct the &Action; object yourself + and pass that, which gives you much more control. + Here's an evolution of the example from above showing this approach: + + + + + +env = Environment() + +def build(target, source, env): + # Whatever it takes to build + return None + +act = Action(build, cmdstr="Building ${SOURCE}") +env.Command('foo.out', 'foo.in', action=act) + + +foo.in + + + + + + Which executes as follows: + + + + + scons -Q + diff --git a/doc/user/output.xml b/doc/user/output.xml index bae018a..cf6571c 100644 --- a/doc/user/output.xml +++ b/doc/user/output.xml @@ -197,7 +197,7 @@ if env['PLATFORM'] == 'win32': -
+
Controlling How &SCons; Prints Build Commands: the <envar>$*COMSTR</envar> Variables -- cgit v0.12 From c9528b3d1d2645dbf9e495b1373b3190253901ed Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Fri, 28 Aug 2020 09:44:00 -0600 Subject: typo fix [skip appveyor] Signed-off-by: Mats Wichmann --- doc/user/builders-commands.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/user/builders-commands.xml b/doc/user/builders-commands.xml index 9bbd962..495f05d 100644 --- a/doc/user/builders-commands.xml +++ b/doc/user/builders-commands.xml @@ -198,11 +198,11 @@ env.Command('${SOURCE.basename}.out', 'foo.in', action=build) have pre-defined *COMSTR variables for that purpose, but that is not the case when calling &f-Command;, where &SCons; has no specific knowledge of the action ahead of time. - If the action argument to &f-Command is not already an &Action; object, + If the action argument to &f-Command; is not already an &Action; object, it will construct one for you with suitable defaults, which include a message based on the type of action. However, you can also construct the &Action; object yourself - and pass that, which gives you much more control. + to pass to &f-Command;, which gives you much more control. Here's an evolution of the example from above showing this approach: -- cgit v0.12 From 390e948c467a6d19f9f3d08db4e4bb90d3bc5a75 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Fri, 28 Aug 2020 09:48:54 -0600 Subject: and fix a "thinko" in new userguide example [skip appveyor] Signed-off-by: Mats Wichmann --- doc/generated/examples/builderscommands_ex5_1.xml | 2 +- doc/user/builders-commands.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/generated/examples/builderscommands_ex5_1.xml b/doc/generated/examples/builderscommands_ex5_1.xml index 3811242..67659c1 100644 --- a/doc/generated/examples/builderscommands_ex5_1.xml +++ b/doc/generated/examples/builderscommands_ex5_1.xml @@ -1,3 +1,3 @@ % scons -Q -Building foo.in +Building foo.out diff --git a/doc/user/builders-commands.xml b/doc/user/builders-commands.xml index 495f05d..7d47dae 100644 --- a/doc/user/builders-commands.xml +++ b/doc/user/builders-commands.xml @@ -215,7 +215,7 @@ def build(target, source, env): # Whatever it takes to build return None -act = Action(build, cmdstr="Building ${SOURCE}") +act = Action(build, cmdstr="Building ${TARGET}") env.Command('foo.out', 'foo.in', action=act) -- cgit v0.12 From fe25f9da4744cbe7d6ebd9de24835fb68564880b Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Fri, 28 Aug 2020 10:25:50 -0600 Subject: [PR #3787] put back a return and update docstring Fix a review comment on dropped return. Update the ParseFlags docstring, which contained a reference to something "below" which only was correct in the context of looking at the function source; there is no "below" if looking at the generated API Doc entry. Signed-off-by: Mats Wichmann --- SCons/Environment.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/SCons/Environment.py b/SCons/Environment.py index c874bd0..76128ed 100644 --- a/SCons/Environment.py +++ b/SCons/Environment.py @@ -651,10 +651,14 @@ class SubstitutionEnvironment: Parse ``flags`` and return a dict with the flags distributed into the appropriate construction variable names. The flags are treated - as a typical set of command-line flags for a GNU-like toolchain and - used to populate the entries in the dict immediately below. - If one of the flag strings begins with a bang (exclamation mark), it is - assumed to be a command and the rest of the string is executed; + as a typical set of command-line flags for a GNU-like toolchain, + such as might have been generated by one of the \*-config scripts, + and used to populate the entries based on knowledge embedded in + this method - the choices are not expected to be portable to other + toolchains. + + If one of the ``flags`` strings begins with a bang (exclamation mark), + it is assumed to be a command and the rest of the string is executed; the result of that evaluation is then added to the dict. """ dict = { @@ -1631,7 +1635,7 @@ class Base(SubstitutionEnvironment): if name[:len(prefix)] == prefix and name[-len(suffix):] == suffix: return path - def ParseConfig(self, command, function=None, unique=1): + def ParseConfig(self, command, function=None, unique=True): """ Use the specified function to parse the output of the command in order to modify the current environment. The 'command' can @@ -1644,14 +1648,14 @@ class Base(SubstitutionEnvironment): """ if function is None: def parse_conf(env, cmd, unique=unique): - env.MergeFlags(cmd, unique) + return env.MergeFlags(cmd, unique) function = parse_conf if SCons.Util.is_List(command): command = ' '.join(command) command = self.subst(command) return function(self, self.backtick(command)) - def ParseDepends(self, filename, must_exist=None, only_one=0): + def ParseDepends(self, filename, must_exist=None, only_one=False): """ Parse a mkdep-style file for explicit dependencies. This is completely abusable, and should be unnecessary in the "normal" -- cgit v0.12 From b85cb312467893c60e07f3732971775967ce8fab Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sat, 29 Aug 2020 08:34:37 -0600 Subject: User Guide: don't link to tigris any more [ci skip] Troubleshooting section had a remaining link to tigris.org. Now points to scons website - not to github, because we don't encourage directly filing a bug before first discussing. Signed-off-by: Mats Wichmann --- doc/user/troubleshoot.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/user/troubleshoot.xml b/doc/user/troubleshoot.xml index 7049deb..3906af6 100644 --- a/doc/user/troubleshoot.xml +++ b/doc/user/troubleshoot.xml @@ -69,8 +69,8 @@ odds are pretty good that someone else will run into the same problem, too. If so, please let the SCons development team know - (preferably by filing a bug report - or feature request at our project pages at tigris.org) + using the contact information at + so that we can use your feedback to try to come up with a better way to help you, and others, get the necessary insight into &SCons; behavior -- cgit v0.12 From 722c383f3869776e45493f842610521140b3abd4 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Wed, 2 Sep 2020 09:01:58 -0600 Subject: Include SideEffect section into the UserGuide [skip appveyor] Existing written doc, but was not included - now added to the Miscellaneous chapter. Some rewriting. Signed-off-by: Mats Wichmann --- doc/generated/examples/sideeffect_simple2_1.xml | 4 + doc/generated/examples/sideeffect_simple_1.xml | 2 +- doc/user/MANIFEST | 1 + doc/user/main.xml | 46 ++---- doc/user/misc.xml | 3 + doc/user/parse_flags_arg.xml | 2 +- doc/user/sideeffect.xml | 204 ++++++++++++++++-------- 7 files changed, 155 insertions(+), 107 deletions(-) create mode 100644 doc/generated/examples/sideeffect_simple2_1.xml diff --git a/doc/generated/examples/sideeffect_simple2_1.xml b/doc/generated/examples/sideeffect_simple2_1.xml new file mode 100644 index 0000000..f60625b --- /dev/null +++ b/doc/generated/examples/sideeffect_simple2_1.xml @@ -0,0 +1,4 @@ +% scons -Q +echo > file1 data1; echo >log updated file1 +Copy("file2", "log") + diff --git a/doc/generated/examples/sideeffect_simple_1.xml b/doc/generated/examples/sideeffect_simple_1.xml index cc94830..f60625b 100644 --- a/doc/generated/examples/sideeffect_simple_1.xml +++ b/doc/generated/examples/sideeffect_simple_1.xml @@ -1,4 +1,4 @@ -% scons -Q --jobs=2 +% scons -Q echo > file1 data1; echo >log updated file1 Copy("file2", "log") diff --git a/doc/user/MANIFEST b/doc/user/MANIFEST index d7237df..5be5aff 100644 --- a/doc/user/MANIFEST +++ b/doc/user/MANIFEST @@ -44,6 +44,7 @@ run.xml scanners.xml sconf.xml separate.xml +sideeffect.xml simple.xml tasks.xml tools.xml diff --git a/doc/user/main.xml b/doc/user/main.xml index 6f516f9..a19df62 100644 --- a/doc/user/main.xml +++ b/doc/user/main.xml @@ -82,7 +82,8 @@ Automatically Putting Command-line Options into their Construction Variables - @@ -106,58 +107,33 @@ - - - + - - - + - - - + - + - + + + + + diff --git a/doc/user/misc.xml b/doc/user/misc.xml index b093629..1b18d2f 100644 --- a/doc/user/misc.xml +++ b/doc/user/misc.xml @@ -625,6 +625,9 @@ env.Command('directory_build_info',
+ + +
Virtual environments (virtualenvs) diff --git a/doc/user/parse_flags_arg.xml b/doc/user/parse_flags_arg.xml index 6e9b926..6014145 100644 --- a/doc/user/parse_flags_arg.xml +++ b/doc/user/parse_flags_arg.xml @@ -17,7 +17,7 @@ xmlns="http://www.scons.org/dbxsd/v1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.scons.org/dbxsd/v1.0 http://www.scons.org/dbxsd/v1.0/scons.xsd"> -Merging Options into the Environment: the <parameter>merge_flags</parameter> Parameter +Merging Options While Creating Environment: the <parameter>parse_flags</parameter> Parameter +variant_dir. -The default behavior is for -&scons; -to physically duplicate the source files in the variant tree. +By default, &SCons; +physically duplicates the source files and SConscript files +as needed into the variant tree. Thus, a build performed in the variant tree is guaranteed to be identical to a build performed in the source tree even if intermediate source files are generated during the build, -or preprocessors or other scanners search for included files +or if preprocessors or other scanners search for included files relative to the source file, -or individual compilers or other invoked tools are hard-coded +or if individual compilers or other invoked tools are hard-coded to put derived files in the same directory as source files. +Only the files &SCons; calculates are needed for the build are +duplicated into variant_dir. If possible on the platform, -the duplication is performed by linking rather than copying; -see also the +the duplication is performed by linking rather than copying. +This behavior is affected by the command-line option. -Moreover, only the files needed for the build are duplicated; -files and directories that are not used are not present in -variant_dir. -Duplicating the source tree may be disabled by setting the -duplicate +Physically duplicating the source files may be disabled by setting the +duplicate argument to -0 -(zero). +False. This will cause -&scons; +&SCons; to invoke Builders using the path names of source files in src_dir and the path names of derived files within variant_dir. -This is always more efficient than -duplicate=1, -and is usually safe for most builds -(but see above for cases that may cause problems). +This is more efficient than +duplicate=True, +and is safe for most builds; +revert to True +if it causes problems. -Note that &f-VariantDir; -works most naturally with a subsidiary SConscript file. -However, you would then call the subsidiary SConscript file -not in the source directory, but in the +works most naturally with used with a subsidiary SConscript file. +The subsidiary SConscript file is called as if it +were in variant_dir, regardless of the value of -duplicate. +duplicate. This is how you tell &scons; which variant of a source tree to build: @@ -3319,15 +3304,11 @@ Examples: # use names in the build directory, not the source directory VariantDir('build', 'src', duplicate=0) Program('build/prog', 'build/source.c') - - # this builds both the source and docs in a separate subtree VariantDir('build', '.', duplicate=0) SConscript(dirs=['build/src','build/doc']) - - # same as previous example, but only uses SConscript SConscript(dirs='src', variant_dir='build/src', duplicate=0) SConscript(dirs='doc', variant_dir='build/doc', duplicate=0) diff --git a/doc/generated/examples/separate_ex1_1.xml b/doc/generated/examples/separate_ex1_1.xml index 6efc16f..803bc4f 100644 --- a/doc/generated/examples/separate_ex1_1.xml +++ b/doc/generated/examples/separate_ex1_1.xml @@ -3,6 +3,8 @@ SConscript hello.c % scons -Q cc -o build/hello.o -c build/hello.c cc -o build/hello build/hello.o +% ls src +SConscript hello.c % ls build SConscript hello hello.c hello.o diff --git a/doc/user/main.xml b/doc/user/main.xml index a19df62..494a208 100644 --- a/doc/user/main.xml +++ b/doc/user/main.xml @@ -105,7 +105,6 @@ - diff --git a/doc/user/separate.xml b/doc/user/separate.xml index 748a124..a000260 100644 --- a/doc/user/separate.xml +++ b/doc/user/separate.xml @@ -2,7 +2,7 @@ %scons; - + %builders-mod; @@ -17,7 +17,7 @@ xmlns="http://www.scons.org/dbxsd/v1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.scons.org/dbxsd/v1.0 http://www.scons.org/dbxsd/v1.0/scons.xsd"> -Separating Source and Build Directories +Separating Source and Build Trees: Variant Directories - + - It's often useful to keep any built files completely - separate from the source files. - In &SCons;, this is usually done by creating one or more separate - variant directory trees + You can enable this separation by creating one or more + variant directory trees that are used to hold the built objects files, libraries, and executable programs, etc. for a specific flavor, or variant, of build. &SCons; provides two ways to do this, - one through the &SConscript; function that we've already seen, - and the second through a more flexible &VariantDir; function. + one through the &f-link-SConscript; function that we've already seen, + and the second through a more flexible &f-link-VariantDir; function. - One historical note: the &VariantDir; function + Historical note: the &VariantDir; function used to be called &BuildDir;, a name which was removed because the &SCons; functionality - differs from the model of a "build directory" - implemented by other build systems like the GNU Autotools. + differs from a familiar model of a "build directory" + implemented by other build systems like GNU Autotools. + You might still find references to the old name on + the Internet in postings about &SCons;, but it no longer works. @@ -162,11 +96,11 @@ program using the F path name. The most straightforward way to establish a variant directory tree - uses the fact that the usual way to + relies the fact that the usual way to set up a build hierarchy is to have an - &SConscript; file in the source subdirectory. - If you then pass a &variant_dir; argument to the - &SConscript; function call: + SConscript file in the source subdirectory. + If you pass a &variant_dir; argument to the + &f-link-SConscript; function call: @@ -193,20 +127,25 @@ int main() { printf("Hello, world!\n"); } ls src scons -Q + ls src ls build - But wait a minute--what's going on here? - &SCons; created the object file + No files were built in &src;, they went to &build;. + The build output might show a bit of a surprise: + the object file build/hello.o - in the &build; subdirectory, + and the executable file + build/hello + were built in the &build; subdirectory, as expected. But even though our &hello_c; file lives in the &src; subdirectory, &SCons; has actually compiled a build/hello.c file - to create the object file. + to create the object file, + and that file is now seen in &build;. @@ -215,7 +154,7 @@ int main() { printf("Hello, world!\n"); } What's happened is that &SCons; has duplicated the &hello_c; file from the &src; subdirectory to the &build; subdirectory, - and built the program from there. + and built the program from there (it also duplicated &SConscript;). The next section explains why &SCons; does this. @@ -227,13 +166,13 @@ int main() { printf("Hello, world!\n"); } - &SCons; duplicates source files in variant directory trees - because it's the most straightforward way to guarantee a correct build - regardless of include-file directory paths, - relative references between files, - or tool support for putting files in different locations, - and the &SCons; philosophy is to, by default, - guarantee a correct build in all cases. + The main thing to understand is that when you set up a variant directory, + &SCons; performs the build in that directory. + It turns out it's easiest to ensure where build products end up + by just building in place. + Since the build is happening in a place different from where the + sources are, the most straightforward way to guarantee a correct build + is for &SCons; to copy them there. @@ -312,13 +251,13 @@ int main() { printf("Hello, world!\n"); } duplicating the source files and everything will work just fine. You can disable the default &SCons; behavior - by specifying duplicate=0 + by specifying duplicate=False when you call the &SConscript; function: -SConscript('src/SConscript', variant_dir='build', duplicate=0) +SConscript('src/SConscript', variant_dir='build', duplicate=False) @@ -395,14 +334,14 @@ int main() { printf("Hello, world!\n"); } - You can specify the same duplicate=0 argument + You can specify the same duplicate=False argument that you can specify for an &SConscript; call: -VariantDir('build', 'src', duplicate=0) +VariantDir('build', 'src', duplicate=False) env = Environment() env.Program('build/hello.c') @@ -432,8 +371,10 @@ int main() { printf("Hello, world!\n"); } Even when using the &VariantDir; function, - it's much more natural to use it with - a subsidiary &SConscript; file. + it's more natural to use it with + a subsidiary &SConscript; file, + because then you don't have to adjust your individual + build instructions to use the variant directory path. For example, if the src/SConscript looks like this: @@ -490,7 +431,7 @@ int main() { printf("Hello, world!\n"); } - The &Glob; file name pattern matching function + The &f-link-Glob; file name pattern matching function works just as usual when using &VariantDir;. For example, if the src/SConscript @@ -558,4 +499,6 @@ const char * f2(); --> + + diff --git a/doc/user/variants.xml b/doc/user/variants.xml index 0c83b04..6cf8c3d 100644 --- a/doc/user/variants.xml +++ b/doc/user/variants.xml @@ -13,11 +13,11 @@ %variables-mod; ]> - -Variant Builds +Variant Build Examples - - The &variant_dir; keyword argument of @@ -76,9 +52,15 @@ is pretty smart about rebuilding things when you change options. variant builds using &SCons;. Suppose, for example, that we want to build a program for both Windows and Linux platforms, - but that we want to build it in a shared directory + but that we want to build it in directory on a network share with separate side-by-side build directories for the Windows and Linux versions of the program. + We have to do a little bit of work to construct paths, + to make sure unwanted location dependencies don't creep in. + The top-relative path reference can be useful here. + To avoid writing conditional code based on platform, + we can build the variant_dir + path dynamically: @@ -90,13 +72,15 @@ include = "#export/$PLATFORM/include" lib = "#export/$PLATFORM/lib" bin = "#export/$PLATFORM/bin" -env = Environment(PLATFORM = platform, - BINDIR = bin, - INCDIR = include, - LIBDIR = lib, - CPPPATH = [include], - LIBPATH = [lib], - LIBS = 'world') +env = Environment( + PLATFORM=platform, + BINDIR=bin, + INCDIR=include, + LIBDIR=lib, + CPPPATH=[include], + LIBPATH=[lib], + LIBS='world', +) Export('env') @@ -155,20 +139,32 @@ int world() { printf "world.c\n"; } scons -Q OS=windows - - +
-- cgit v0.12 From d0a3e4ed38f4a4cd84000dec2342f51b6dda69b9 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sat, 11 Jul 2020 08:06:16 -0600 Subject: Update some copyright strings and drop __revision__ [skip appveyor] Touches the first and second levels of SCons (except SCons.Tool), not tests or docs which remain TODO. Make sure docstring is first non-comment content, eliminate cases where docstring is set elsewhere but assigns to __doc__ - this approach of course worked inside Python, but confuses various tools. Some module-level docstrings modified a bit, in particular the convention of having the name of the module as the first line is dropped, replaced by a summary description going there instead - this improves the look in the API Docs, which otherwise display something like: SCons.Foo - SCons.Foo Signed-off-by: Mats Wichmann --- SCons/Action.py | 54 +++++----- SCons/Action.xml | 2 +- SCons/ActionTests.py | 7 +- SCons/Builder.py | 47 +++++---- SCons/BuilderTests.py | 5 +- SCons/CacheDir.py | 9 +- SCons/CacheDirTests.py | 6 +- SCons/Conftest.py | 145 +++++++++++++-------------- SCons/Debug.py | 22 ++--- SCons/Defaults.py | 28 +++--- SCons/Defaults.xml | 2 +- SCons/DefaultsTests.py | 6 +- SCons/Environment.py | 22 ++--- SCons/Environment.xml | 2 +- SCons/EnvironmentTests.py | 5 +- SCons/EnvironmentValues.py | 23 +++++ SCons/EnvironmentValuesTest.py | 23 +++++ SCons/Errors.py | 11 +-- SCons/ErrorsTests.py | 6 +- SCons/Executor.py | 13 +-- SCons/ExecutorTests.py | 6 +- SCons/Job.py | 18 ++-- SCons/JobTests.py | 5 +- SCons/Memoize.py | 7 +- SCons/MemoizeTests.py | 6 +- SCons/Node/Alias.py | 18 ++-- SCons/Node/AliasTests.py | 6 +- SCons/Node/FS.py | 29 +++--- SCons/Node/FSTests.py | 6 +- SCons/Node/NodeTests.py | 5 +- SCons/Node/Python.py | 12 +-- SCons/Node/PythonTests.py | 6 +- SCons/Node/__init__.py | 43 ++++---- SCons/PathList.py | 17 ++-- SCons/PathListTests.py | 6 +- SCons/Platform/Platform.xml | 2 +- SCons/Platform/PlatformTests.py | 7 +- SCons/Platform/__init__.py | 42 ++++---- SCons/Platform/aix.py | 20 ++-- SCons/Platform/cygwin.py | 20 ++-- SCons/Platform/darwin.py | 20 ++-- SCons/Platform/hpux.py | 20 ++-- SCons/Platform/irix.py | 20 ++-- SCons/Platform/mingw.py | 14 +-- SCons/Platform/os2.py | 21 ++-- SCons/Platform/posix.py | 20 ++-- SCons/Platform/posix.xml | 2 +- SCons/Platform/sunos.py | 20 ++-- SCons/Platform/sunos.xml | 2 +- SCons/Platform/virtualenv.py | 15 +-- SCons/Platform/virtualenvTests.py | 9 +- SCons/Platform/win32.py | 20 ++-- SCons/Platform/win32.xml | 2 +- SCons/SConf.py | 28 +++--- SCons/SConfTests.py | 6 +- SCons/SConsign.py | 13 +-- SCons/SConsignTests.py | 6 +- SCons/Scanner/C.py | 35 +++---- SCons/Scanner/CTests.py | 9 +- SCons/Scanner/D.py | 18 ++-- SCons/Scanner/DTests.py | 10 +- SCons/Scanner/Dir.py | 5 +- SCons/Scanner/DirTests.py | 6 +- SCons/Scanner/Fortran.py | 11 +-- SCons/Scanner/FortranTests.py | 7 +- SCons/Scanner/IDL.py | 23 ++--- SCons/Scanner/IDLTests.py | 6 +- SCons/Scanner/LaTeX.py | 18 ++-- SCons/Scanner/LaTeXTests.py | 9 +- SCons/Scanner/Prog.py | 16 ++- SCons/Scanner/ProgTests.py | 6 +- SCons/Scanner/Python.py | 31 +++--- SCons/Scanner/PythonTests.py | 6 +- SCons/Scanner/RC.py | 38 +++----- SCons/Scanner/RCTests.py | 6 +- SCons/Scanner/SWIG.py | 12 +-- SCons/Scanner/Scanner.xml | 2 +- SCons/Scanner/ScannerTests.py | 8 +- SCons/Scanner/__init__.py | 168 ++++++++++++++++---------------- SCons/Script/Interactive.py | 8 +- SCons/Script/Main.py | 33 +++---- SCons/Script/Main.xml | 2 +- SCons/Script/MainTests.py | 6 +- SCons/Script/SConsOptions.py | 6 +- SCons/Script/SConscript.py | 12 +-- SCons/Script/SConscript.xml | 2 +- SCons/Script/SConscriptTests.py | 6 +- SCons/Script/__init__.py | 34 +++---- SCons/Subst.py | 24 ++--- SCons/Subst.xml | 2 +- SCons/SubstTests.py | 5 +- SCons/Taskmaster.py | 57 +++++------ SCons/TaskmasterTests.py | 5 +- SCons/Util.py | 12 +-- SCons/UtilTests.py | 6 +- SCons/Utilities/ConfigureCache.py | 6 +- SCons/Utilities/sconsign.py | 6 +- SCons/Variables/BoolVariable.py | 28 +++--- SCons/Variables/BoolVariableTests.py | 6 +- SCons/Variables/EnumVariable.py | 44 +++++---- SCons/Variables/EnumVariableTests.py | 6 +- SCons/Variables/ListVariable.py | 59 +++++------ SCons/Variables/ListVariableTests.py | 6 +- SCons/Variables/PackageVariable.py | 54 +++++----- SCons/Variables/PackageVariableTests.py | 6 +- SCons/Variables/PathVariable.py | 78 +++++++-------- SCons/Variables/PathVariableTests.py | 6 +- SCons/Variables/VariablesTests.py | 6 +- SCons/Variables/__init__.py | 11 +-- SCons/Warnings.py | 12 +-- SCons/WarningsTests.py | 6 +- SCons/__main__.py | 23 +++++ SCons/compat/__init__.py | 19 ++-- SCons/compat/_scons_dbm.py | 9 +- SCons/cpp.py | 11 +-- SCons/cppTests.py | 6 +- SCons/dblite.py | 23 +++++ SCons/exitfuncs.py | 13 +-- 118 files changed, 926 insertions(+), 1145 deletions(-) diff --git a/SCons/Action.py b/SCons/Action.py index 1f499ee..d7f10df 100644 --- a/SCons/Action.py +++ b/SCons/Action.py @@ -1,11 +1,34 @@ -"""SCons.Action +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +"""SCons Actions. -This encapsulates information about executing any sort of action that +Information about executing any sort of action that can build one or more target Nodes (typically files) from one or more source Nodes (also typically files) given a specific Environment. The base class here is ActionBase. The base class supplies just a few -OO utility methods and some generic methods for displaying information +utility methods and some generic methods for displaying information about an Action in response to the various commands that control printing. A second-level base class is _ActionAction. This extends ActionBase @@ -60,7 +83,7 @@ this module: get_presig() Fetches the "contents" of a subclass for signature calculation. The varlist is added to this to produce the Action's contents. - TODO(?): Change this to always return ascii/bytes and not unicode (or py3 strings) + TODO(?): Change this to always return bytes and not str? strfunction() Returns a substituted string representation of the Action. @@ -77,29 +100,6 @@ way for wrapping up the functions. """ -# __COPYRIGHT__ -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - import os import pickle import re diff --git a/SCons/Action.xml b/SCons/Action.xml index 8263a57..0e3ef33 100644 --- a/SCons/Action.xml +++ b/SCons/Action.xml @@ -1,6 +1,6 @@ - + -- cgit v0.12 From 0378c0f0071421cb176d61eb060cef492c6d1ac1 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 3 Oct 2020 17:04:06 -0700 Subject: Remove using pywin32 to retrieve peak memory usage for --debug=memory --- CHANGES.txt | 1 + SCons/Debug.py | 20 ++++------ SCons/compat/win32.py | 101 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 109 insertions(+), 13 deletions(-) create mode 100644 SCons/compat/win32.py diff --git a/CHANGES.txt b/CHANGES.txt index 6ca145e..b619526 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -17,6 +17,7 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER From William Deegan: - Fix yacc tool, not respecting YACC set at time of tool initialization. - Refactor SCons.Tool to move all common shared and loadable module linking logic to SCons.Tool.linkCommon + - Remove using pywin32 to retrieve peak memory usage on Win32 for `--debug=memory` From Michał Górny: - Fix dvipdf test failure due to passing incorrect flag to dvipdf. diff --git a/SCons/Debug.py b/SCons/Debug.py index b93617c..2310cb7 100644 --- a/SCons/Debug.py +++ b/SCons/Debug.py @@ -90,7 +90,6 @@ def dumpLoggedInstances(classes, file=sys.stdout): file.write(' %20s : %s\n' % (key, value)) - if sys.platform[:5] == "linux": # Linux doesn't actually support memory usage stats from getrusage(). def memory(): @@ -102,28 +101,23 @@ elif sys.platform[:6] == 'darwin': #TODO really get memory stats for OS X def memory(): return 0 +elif sys.platform == 'win32': + from SCons.compat.win32 import get_peak_memory_usage + memory = get_peak_memory_usage else: try: import resource except ImportError: - try: - import win32process - import win32api - except ImportError: - def memory(): - return 0 - else: - def memory(): - process_handle = win32api.GetCurrentProcess() - memory_info = win32process.GetProcessMemoryInfo( process_handle ) - return memory_info['PeakWorkingSetSize'] + def memory(): + return 0 else: def memory(): res = resource.getrusage(resource.RUSAGE_SELF) return res[4] -# returns caller's stack + def caller_stack(): + """return caller's stack""" import traceback tb = traceback.extract_stack() # strip itself and the caller from the output diff --git a/SCons/compat/win32.py b/SCons/compat/win32.py new file mode 100644 index 0000000..e01adfa --- /dev/null +++ b/SCons/compat/win32.py @@ -0,0 +1,101 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Logic to replicate parts of pywin32 SCons uses.""" + +__all__ = ['get_current_process', 'get_memory_info', 'get_memory_usage', 'get_peak_memory_usage'] + +import ctypes +from ctypes import wintypes + +# +# From Activestate Recipe +# https://code.activestate.com/recipes/578513-get-memory-usage-of-windows-processes-using-getpro/ +# MIT licensed +# +GetCurrentProcess = ctypes.windll.kernel32.GetCurrentProcess +GetCurrentProcess.argtypes = [] +GetCurrentProcess.restype = wintypes.HANDLE + +SIZE_T = ctypes.c_size_t + + +class PROCESS_MEMORY_COUNTERS_EX(ctypes.Structure): + _fields_ = [ + ('cb', wintypes.DWORD), + ('PageFaultCount', wintypes.DWORD), + ('PeakWorkingSetSize', SIZE_T), + ('WorkingSetSize', SIZE_T), + ('QuotaPeakPagedPoolUsage', SIZE_T), + ('QuotaPagedPoolUsage', SIZE_T), + ('QuotaPeakNonPagedPoolUsage', SIZE_T), + ('QuotaNonPagedPoolUsage', SIZE_T), + ('PagefileUsage', SIZE_T), + ('PeakPagefileUsage', SIZE_T), + ('PrivateUsage', SIZE_T), + ] + + +GetProcessMemoryInfo = ctypes.windll.psapi.GetProcessMemoryInfo +GetProcessMemoryInfo.argtypes = [ + wintypes.HANDLE, + ctypes.POINTER(PROCESS_MEMORY_COUNTERS_EX), + wintypes.DWORD, +] +GetProcessMemoryInfo.restype = wintypes.BOOL + + +def get_current_process(): + """Return handle to current process.""" + return GetCurrentProcess() + + +def get_memory_info(process=None): + """Return Win32 process memory counters structure as a dict.""" + if process is None: + process = get_current_process() + counters = PROCESS_MEMORY_COUNTERS_EX() + ret = GetProcessMemoryInfo(process, ctypes.byref(counters), + ctypes.sizeof(counters)) + if not ret: + raise ctypes.WinError() + info = dict((name, getattr(counters, name)) + for name, _ in counters._fields_) + return info + + +def get_memory_usage(process=None): + """Return this process's memory usage in bytes.""" + info = get_memory_info(process=process) + return info['PrivateUsage'] + + +def get_peak_memory_usage(process=None): + """Return this process's memory usage in bytes.""" + info = get_memory_info(process=process) + return info['PeakWorkingSetSize'] + + +if __name__ == '__main__': + import pprint + + pprint.pprint(get_memory_info()) -- cgit v0.12 From 6f5a2b1d63620da702ffee2d35746a6d0dfa8bf2 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 3 Oct 2020 17:13:12 -0700 Subject: Remove pywin32 from SCons.Util where it was used for reading registry if not available via Python's winreg package. It's (long been) part of Python's standard library so using pywin32 as a backup is not necessary --- CHANGES.txt | 2 ++ SCons/Util.py | 19 +++---------------- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 6ca145e..8b8e27b 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -17,6 +17,8 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER From William Deegan: - Fix yacc tool, not respecting YACC set at time of tool initialization. - Refactor SCons.Tool to move all common shared and loadable module linking logic to SCons.Tool.linkCommon + - Remove pywin32 usage from SCons.Util where it was used for accessing the registry. Python native winreg + library already includes this functionality. From Michał Górny: - Fix dvipdf test failure due to passing incorrect flag to dvipdf. diff --git a/SCons/Util.py b/SCons/Util.py index 19b7fbb..347395f 100644 --- a/SCons/Util.py +++ b/SCons/Util.py @@ -648,22 +648,9 @@ try: RegError = winreg.error except ImportError: - try: - import win32api - import win32con - can_read_reg = 1 - hkey_mod = win32con - - RegOpenKeyEx = win32api.RegOpenKeyEx - RegEnumKey = win32api.RegEnumKey - RegEnumValue = win32api.RegEnumValue - RegQueryValueEx = win32api.RegQueryValueEx - RegError = win32api.error - - except ImportError: - class _NoError(Exception): - pass - RegError = _NoError + class _NoError(Exception): + pass + RegError = _NoError # Make sure we have a definition of WindowsError so we can -- cgit v0.12 From cece33add5c095c70d27c8e5897daaac90ff382d Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 3 Oct 2020 17:23:51 -0700 Subject: Remove pywin32 from scons win32 install requirements. Remove usage from SCons.Script.main --- CHANGES.txt | 2 ++ SCons/Platform/win32.py | 16 ---------------- SCons/Script/Main.py | 15 +-------------- setup.cfg | 1 - 4 files changed, 3 insertions(+), 31 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 6ca145e..00315c7 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -17,6 +17,8 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER From William Deegan: - Fix yacc tool, not respecting YACC set at time of tool initialization. - Refactor SCons.Tool to move all common shared and loadable module linking logic to SCons.Tool.linkCommon + - Remove pywin32 imports from SCons.Script.Main. No longer needed. + - pywin32 no longer necessary for SCons install. (pip install SCons will no longer also require pywin32 on win32) From Michał Górny: - Fix dvipdf test failure due to passing incorrect flag to dvipdf. diff --git a/SCons/Platform/win32.py b/SCons/Platform/win32.py index 439cf64..aa76387 100644 --- a/SCons/Platform/win32.py +++ b/SCons/Platform/win32.py @@ -43,22 +43,6 @@ CHOCO_DEFAULT_PATH = [ r'C:\ProgramData\chocolatey\bin' ] -try: - import msvcrt - import win32api - import win32con -except ImportError: - parallel_msg = \ - "you do not seem to have the pywin32 extensions installed;\n" + \ - "\tparallel (-j) builds may not work reliably with open Python files." -except AttributeError: - parallel_msg = \ - "your pywin32 extensions do not support file handle operations;\n" + \ - "\tparallel (-j) builds may not work reliably with open Python files." -else: - parallel_msg = None - - if False: # Now swap out shutil.filecopy and filecopy2 for win32 api native CopyFile try: diff --git a/SCons/Script/Main.py b/SCons/Script/Main.py index 2b9bb90..6c3ba06 100644 --- a/SCons/Script/Main.py +++ b/SCons/Script/Main.py @@ -80,17 +80,6 @@ num_jobs = None delayed_warnings = [] -def fetch_win32_parallel_msg(): - # A subsidiary function that exists solely to isolate this import - # so we don't have to pull it in on all platforms, and so that an - # in-line "import" statement in the _main() function below doesn't - # cause warnings about local names shadowing use of the 'SCons' - # global in nest scopes and UnboundLocalErrors and the like in some - # versions (2.1) of Python. - import SCons.Platform.win32 - return SCons.Platform.win32.parallel_msg - - def revert_io(): # This call is added to revert stderr and stdout to the original # ones just in case some build rule or something else in the system @@ -1288,9 +1277,7 @@ def _build_targets(fs, options, targets, target_top): jobs = SCons.Job.Jobs(num_jobs, taskmaster) if num_jobs > 1: msg = None - if sys.platform == 'win32': - msg = fetch_win32_parallel_msg() - elif jobs.num_jobs == 1 or not python_has_threads: + if jobs.num_jobs == 1 or not python_has_threads: msg = "parallel builds are unsupported by this version of Python;\n" + \ "\tignoring -j or num_jobs option.\n" if msg: diff --git a/setup.cfg b/setup.cfg index 1243fd2..a141c67 100644 --- a/setup.cfg +++ b/setup.cfg @@ -42,7 +42,6 @@ zip_safe = False python_requires = >=3.5 install_requires = setuptools - pywin32 >= 1.0;platform_system=="Windows" setup_requires = setuptools include_package_data = True -- cgit v0.12 From 06c0204e85ecbb124e72b27f4d23483ab3bd7c35 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 3 Oct 2020 17:27:37 -0700 Subject: remove win32api from test --- test/option/debug-memory.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/test/option/debug-memory.py b/test/option/debug-memory.py index bf720cc..9eefe65 100644 --- a/test/option/debug-memory.py +++ b/test/option/debug-memory.py @@ -1,6 +1,7 @@ #!/usr/bin/env python +# MIT License # -# __COPYRIGHT__ +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -22,8 +23,6 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - """ Test that the --debug=memory option works. """ @@ -34,18 +33,14 @@ import TestSCons test = TestSCons.TestSCons() -try: - import resource -except ImportError: +if not test.IS_WINDOWS: try: - import win32process - import win32api + import resource except ImportError: - x = "Python version has no 'resource' or 'win32api/win32process' module; skipping tests.\n" + x = "Python version has no 'resource' skipping tests.\n" test.skip_test(x) - test.write('SConstruct', """ DefaultEnvironment(tools=[]) def cat(target, source, env): -- cgit v0.12 From 0ca47bd580b3bc93bbff4bbaaa7f0b3dad160558 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 3 Oct 2020 18:22:01 -0700 Subject: Disable flake8 F401 in test --- test/option/debug-memory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/option/debug-memory.py b/test/option/debug-memory.py index 9eefe65..31ae346 100644 --- a/test/option/debug-memory.py +++ b/test/option/debug-memory.py @@ -35,7 +35,7 @@ test = TestSCons.TestSCons() if not test.IS_WINDOWS: try: - import resource + import resource # noqa: F401 except ImportError: x = "Python version has no 'resource' skipping tests.\n" test.skip_test(x) -- cgit v0.12 From 643b9e4488eaefe3cfe34df2af7c9f040db32659 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 3 Oct 2020 18:25:38 -0700 Subject: Fix IS_WINDOWS usage for test --- test/option/debug-memory.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/option/debug-memory.py b/test/option/debug-memory.py index 31ae346..2958e0b 100644 --- a/test/option/debug-memory.py +++ b/test/option/debug-memory.py @@ -30,10 +30,11 @@ Test that the --debug=memory option works. import re import TestSCons +from TestCmd import IS_WINDOWS test = TestSCons.TestSCons() -if not test.IS_WINDOWS: +if not IS_WINDOWS: try: import resource # noqa: F401 except ImportError: -- cgit v0.12 From 2793ade9a03c621c28310f98348da2d7a2be6fb1 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sun, 4 Oct 2020 09:58:55 -0700 Subject: Try new way to detect thread support. Check threading.get_ident() which should always return a positive integer. The fake threading module dummy_threading will always return -1 --- SCons/Script/Main.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/SCons/Script/Main.py b/SCons/Script/Main.py index 6c3ba06..1f2a455 100644 --- a/SCons/Script/Main.py +++ b/SCons/Script/Main.py @@ -46,6 +46,7 @@ import time import traceback import sysconfig import platform +import threading import SCons.CacheDir import SCons.Debug @@ -1270,7 +1271,12 @@ def _build_targets(fs, options, targets, target_top): # As of 3.7, python removed support for threadless platforms. # See https://www.python.org/dev/peps/pep-0011/ is_37_or_later = sys.version_info >= (3, 7) - python_has_threads = sysconfig.get_config_var('WITH_THREAD') or is_pypy or is_37_or_later + # python_has_threads = sysconfig.get_config_var('WITH_THREAD') or is_pypy or is_37_or_later + + # As of python 3.4 threading has a dummy_threading module for use when there is no threading + # it's get_ident() will allways return -1, while real threading modules get_ident() will + # always return a positive integer + python_has_threads = threading.get_ident() != -1 # to check if python configured with threads. global num_jobs num_jobs = options.num_jobs -- cgit v0.12 From 1a14204dee6840242196b25f5a4433eac0b5249c Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sat, 3 Oct 2020 14:19:00 -0600 Subject: Further runtest tweaks Drop unused package and builddir options. Extract the reading of a list of tests to a common routine. Move the e2e-only and unit-only to a separate postprocessing step, and combine with exclude processing. Signed-off-by: Mats Wichmann --- runtest.py | 101 ++++++++++++++++++++++--------------------------------------- 1 file changed, 36 insertions(+), 65 deletions(-) diff --git a/runtest.py b/runtest.py index 87d9a3f..16c4f0f 100755 --- a/runtest.py +++ b/runtest.py @@ -6,22 +6,17 @@ # # The SCons test suite consists of: # -# - unit tests - included in *Tests.py files from src/ dir +# - unit tests - included in *Tests.py files from SCons/ dir # - end-to-end tests - these are *.py files in test/ directory that # require custom SCons framework from testing/ # # This script adds SCons/ and testing/ directories to PYTHONPATH, # performs test discovery and processes them according to options. -# -# With -p (--package) option, script tests specified package from -# build directory and sets PYTHONPATH to reference modules unpacked -# during build process for testing purposes (build/test-*). """ Options: -a --all Run all tests. -b --baseline BASE Run test scripts against baseline BASE. - --builddir DIR Directory in which packages were built. -d --debug Run test scripts under the Python debugger. -D --devmode Run tests in Python's development mode (3.7+ only) --e2e-only Run only the end-to-end tests @@ -38,15 +33,6 @@ Options: chars! You might run into some deadlocks else. -o --output FILE Save the output from a test run to the log file. -P PYTHON Use the specified Python interpreter. - -p --package PACKAGE Test against the specified PACKAGE: - deb Debian - local-tar-gz .tar.gz standalone package - local-zip .zip standalone package - rpm Red Hat - src-tar-gz .tar.gz source package - src-zip .zip source package - tar-gz .tar.gz distribution - zip .zip distribution --passed Summarize which tests passed. -q --quiet Don't print the test being executed. --quit-on-failure Quit on any test failure. @@ -89,7 +75,6 @@ from queue import Queue cwd = os.getcwd() baseline = None -builddir = os.path.join(cwd, 'build') external = 0 devmode = False debug = '' @@ -97,7 +82,6 @@ execute_tests = True jobs = 1 list_only = False printcommand = True -package = None print_passed_summary = False scons = None scons_exec = False @@ -105,7 +89,6 @@ testlistfile = None version = '' print_times = False python = None -sp = [] print_progress = True catch_output = False suppress_output = False @@ -156,7 +139,6 @@ opts, args = getopt.getopt( "b:dDef:hj:klnP:p:qsv:Xx:t", [ "baseline=", - "builddir=", "debug", "devmode", "external", @@ -167,7 +149,6 @@ opts, args = getopt.getopt( "list", "no-exec", "nopipefiles", - "package=", "passed", "python=", "quiet", @@ -186,10 +167,6 @@ opts, args = getopt.getopt( for o, a in opts: if o in ['-b', '--baseline']: baseline = a - elif o in ['--builddir']: - builddir = a - if not os.path.isabs(builddir): - builddir = os.path.normpath(os.path.join(cwd, builddir)) elif o in ['-d', '--debug']: for d in sys.path: pdb = os.path.join(d, 'pdb.py') @@ -220,8 +197,6 @@ for o, a in opts: execute_tests = False elif o in ['--nopipefiles']: allow_pipe_files = False - elif o in ['-p', '--package']: - package = a elif o in ['--passed']: print_passed_summary = True elif o in ['-P', '--python']: @@ -252,7 +227,7 @@ for o, a in opts: unit_only = True -class Unbuffered(): +class Unbuffered: """ class to arrange for stdout/stderr to be unbuffered """ def __init__(self, file): self.file = file @@ -270,7 +245,7 @@ sys.stderr = Unbuffered(sys.stderr) if options.output: logfile = open(options.output, 'w') - class Tee(): + class Tee: def __init__(self, openfile, stream): self.file = openfile self.stream = stream @@ -305,13 +280,9 @@ else: return f return None -sp.append(builddir) -sp.append(cwd) -# _ws = re.compile(r'\s') - def escape(s): if _ws.search(s): s = '"' + s + '"' @@ -423,7 +394,7 @@ class PopenExecutor(RuntestBase): by calling subprocess.run (behind the covers uses Popen. Very similar to SystemExecutor, but uses command_str instead of command_args, and doesn't allow for not catching - the output. + the output). """ # For an explanation of the following 'if ... else' # and the 'allow_pipe_files' option, please check out the @@ -571,25 +542,29 @@ if old_pythonpath: # # Each test path, whichever of the three sources it comes from, # specifies either a test file or a directory to search for -# SCons tests. SCons code layout assumes that any file under the 'src' +# SCons tests. SCons code layout assumes that any file under the 'SCons' # subdirectory that ends with 'Tests.py' is a unit test, and any Python # script (*.py) under the 'test' subdirectory is an end-to-end test. # We need to track these because they are invoked differently. # find_unit_tests and find_e2e_tests are used for this searching. # -# Note that there are some tests under 'src' that *begin* with +# Note that there are some tests under 'SCons' that *begin* with # 'test_', but they're packaging and installation tests, not # functional tests, so we don't execute them by default. (They can # still be executed by hand, though). # # Test exclusions, if specified, are then applied. -tests = [] -excludetests = [] unittests = [] endtests = [] +def scanlist(testlist): + """Process a testlist file""" + tests = [t.strip() for t in testlist if not t.startswith('#')] + return [t for t in tests if t] + + def find_unit_tests(directory): """ Look for unit tests """ result = [] @@ -613,7 +588,7 @@ def find_e2e_tests(directory): continue try: with open(os.path.join(dirpath, ".exclude_tests")) as f: - excludes = [e.split("#", 1)[0].strip() for e in f.readlines()] + excludes = scanlist(f) except EnvironmentError: excludes = [] for fname in filenames: @@ -622,10 +597,10 @@ def find_e2e_tests(directory): return sorted(result) +# initial selection: if testlistfile: with open(testlistfile, 'r') as f: - tests = [x[:-1].strip() for x in f if not x.startswith('#')] - tests = [x for x in tests if x] + tests = scanlist(f) else: testpaths = [] if options.all: @@ -644,41 +619,37 @@ else: # sys.stderr.write("to:%s\n"%tp) for path in glob.glob(tp): if os.path.isdir(path): - if path.startswith(('SCons', 'testing')) and not e2e_only: - for p in find_unit_tests(path): - unittests.append(p) - elif path.startswith('test') and not unit_only: - for p in find_e2e_tests(path): - endtests.append(p) + if path.startswith(('SCons', 'testing')): + unittests.extend(find_unit_tests(path)) + elif path.startswith('test'): + endtests.extend(find_e2e_tests(path)) else: - if path.endswith("Tests.py") and not e2e_only: + if path.endswith("Tests.py"): unittests.append(path) - else: - if not unit_only: - endtests.append(path) - - tests.extend(unittests) - tests.extend(endtests) - tests.sort() + elif path.endswith(".py"): + endtests.append(path) + tests = unittests + endtests + +# Remove exclusions: +if e2e_only: + tests = [t for t in tests if not t.endswith("Tests.py")] +if unit_only: + tests = [t for t in tests if t.endswith("Tests.py")] +if excludelistfile: + with open(excludelistfile, 'r') as f: + excludetests = scanlist(f) + tests = [t for t in tests if t not in excludetests] if not tests: sys.stderr.write(usagestr + """ -runtest.py: No tests were found. - Tests can be specified on the command line, read from file - with -f option, or discovered with -a to run all tests. +runtest: no tests were found. + Tests can be specified on the command line, read from a file with + the -f/--file option, or discovered with -a/--all to run all tests. """) sys.exit(1) -if excludelistfile: - with open(excludelistfile, 'r') as f: - excludetests = f.readlines() - excludetests = [x for x in excludetests if x[0] != '#'] - excludetests = [x[:-1] for x in excludetests] - excludetests = [x.strip() for x in excludetests] - excludetests = [x for x in excludetests if x] # ---[ test processing ]----------------------------------- -tests = [t for t in tests if t not in excludetests] tests = [Test(t, n + 1) for n, t in enumerate(tests)] if list_only: -- cgit v0.12 From e527ba58ae5ee32e8112e7b6e8855efe2dfb02df Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sun, 4 Oct 2020 13:21:27 -0700 Subject: Remove references to needing pywin32 from docs and readmes --- README-SF.rst | 3 +-- README-package.rst | 3 +-- README.rst | 3 +-- doc/man/scons.xml | 4 +--- 4 files changed, 4 insertions(+), 9 deletions(-) diff --git a/README-SF.rst b/README-SF.rst index e33c3f1..fce28d9 100755 --- a/README-SF.rst +++ b/README-SF.rst @@ -48,8 +48,7 @@ Execution Requirements ====================== Running SCons requires Python 3.5 or higher. There should be no other -dependencies or requirements to run scons, although the pywin32 Python -package is strongly recommended if running on Windows systems. +dependencies or requirements to run scons. The default SCons configuration assumes use of the Microsoft Visual C++ compiler suite on Win32 systems, and assumes a C compiler named 'cc', a C++ diff --git a/README-package.rst b/README-package.rst index 605e5fe..4aaea1c 100755 --- a/README-package.rst +++ b/README-package.rst @@ -45,8 +45,7 @@ Requirements ------------ Running SCons requires Python 3.5 or higher. There should be no other -dependencies or requirements to run SCons, although the pywin32 Python -package is strongly recommended if running on Windows systems. +dependencies or requirements to run SCons. By default, SCons knows how to search for available programming tools on diff --git a/README.rst b/README.rst index df74a59..1d7b858 100755 --- a/README.rst +++ b/README.rst @@ -73,8 +73,7 @@ Execution Requirements ====================== Running SCons requires Python 3.5 or higher. There should be no other -dependencies or requirements to run scons, although the pywin32 Python -package is strongly recommended if running on Windows systems. +dependencies or requirements to run scons The default SCons configuration assumes use of the Microsoft Visual C++ compiler suite on Win32 systems, and assumes a C compiler named 'cc', a C++ diff --git a/doc/man/scons.xml b/doc/man/scons.xml index 2e8a4f1..c930eb6 100644 --- a/doc/man/scons.xml +++ b/doc/man/scons.xml @@ -515,9 +515,7 @@ by appropriate setting of &consvars;. &scons; requires Python 3.5 or higher. -There should be no other dependencies or requirements to run &scons;, -although the pywin32 Python package is -strongly recommended if running on Windows systems. +There should be no other dependencies or requirements to run &scons;. -- cgit v0.12 From 4f496d1e577e9647514ceb8da34d446650126b9d Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sun, 4 Oct 2020 13:22:55 -0700 Subject: Remove pywin32 from appveyor setup files --- .appveyor/install.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.appveyor/install.bat b/.appveyor/install.bat index 67a064d..3dea66a 100644 --- a/.appveyor/install.bat +++ b/.appveyor/install.bat @@ -3,7 +3,7 @@ for /F "tokens=*" %%g in ('C:\\%WINPYTHON%\\python.exe -c "import sys; print(sys REM use mingw 32 bit until #3291 is resolved set PATH=C:\\%WINPYTHON%;C:\\%WINPYTHON%\\Scripts;C:\\ProgramData\\chocolatey\\bin;C:\\MinGW\\bin;C:\\MinGW\\msys\\1.0\\bin;C:\\cygwin\\bin;C:\\msys64\\usr\\bin;C:\\msys64\\mingw64\\bin;%PATH% C:\\%WINPYTHON%\\python.exe -m pip install -U --progress-bar off pip setuptools wheel -C:\\%WINPYTHON%\\python.exe -m pip install -U --progress-bar off pypiwin32 coverage codecov +C:\\%WINPYTHON%\\python.exe -m pip install -U --progress-bar off coverage codecov set STATIC_DEPS=true & C:\\%WINPYTHON%\\python.exe -m pip install -U --progress-bar off lxml REM install 3rd party tools to test with choco install --allow-empty-checksums dmd ldc swig vswhere xsltproc winflexbison -- cgit v0.12 From 8334884bce7d615d045a3daf8bb1a7781d5da524 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sun, 4 Oct 2020 15:15:15 -0700 Subject: [ci skip] Update CHANGES.txt with reference to Issue being fixed #2291 --- CHANGES.txt | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index b9632ea..cdf57de 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -18,10 +18,11 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Fix yacc tool, not respecting YACC set at time of tool initialization. - Refactor SCons.Tool to move all common shared and loadable module linking logic to SCons.Tool.linkCommon - Remove pywin32 imports from SCons.Script.Main. No longer needed. - - pywin32 no longer necessary for SCons install. (pip install SCons will no longer also require pywin32 on win32) - - Remove pywin32 usage from SCons.Util where it was used for accessing the registry. Python native winreg - library already includes this functionality. - - Remove using pywin32 to retrieve peak memory usage on Win32 for `--debug=memory` + - Switch to use ctypes instead of pywin32 (requiring an extra pip install) - Fixes Github Issue #2291 + - pywin32 no longer necessary for SCons install. (pip install SCons will no longer also require pywin32 on win32) + - Remove pywin32 usage from SCons.Util where it was used for accessing the registry. Python native winreg + library already includes this functionality. + - Remove using pywin32 to retrieve peak memory usage on Win32 for `--debug=memory` From Michał Górny: - Fix dvipdf test failure due to passing incorrect flag to dvipdf. -- cgit v0.12 From 1898eb04782848cdf37c8f12e178f85cf6d55d0e Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Wed, 30 Sep 2020 08:01:24 -0600 Subject: Update manpage on variable substitution [skip appveyor] Signed-off-by: Mats Wichmann --- doc/man/scons.xml | 154 +++++++++++++++++++++++++++++++++--------------------- 1 file changed, 93 insertions(+), 61 deletions(-) diff --git a/doc/man/scons.xml b/doc/man/scons.xml index c930eb6..accdae7 100644 --- a/doc/man/scons.xml +++ b/doc/man/scons.xml @@ -2918,16 +2918,22 @@ target file's directory. # scons will change to the "sub" subdirectory # before executing the "cp" command. -env.Command('sub/dir/foo.out', 'sub/dir/foo.in', - "cp dir/foo.in dir/foo.out", - chdir='sub') +env.Command( + target='sub/dir/foo.out', + source='sub/dir/foo.in', + action="cp dir/foo.in dir/foo.out", + chdir='sub', +) # Because chdir is not a string, scons will change to the # target's directory ("sub/dir") before executing the # "cp" command. -env.Command('sub/dir/foo.out', 'sub/dir/foo.in', - "cp foo.in foo.out", - chdir=1) +env.Command( + target='sub/dir/foo.out', + source='sub/dir/foo.in', + action="cp foo.in foo.out", + chdir=True, +) Note that &SCons; will @@ -6182,14 +6188,25 @@ env.Command('marker', 'input_file', action=[MyBuildAction, Touch('$TARGET')]) Before executing a command, &scons; -performs &consvar; substitution on the string that makes up -the command line of the builder. -&Consvars; to be interpolated are indicated in the +performs variable substitution on the string that makes up +the action part of the builder. +Variables to be interpolated are indicated in the string with a leading $, to distinguish them from plain text which is not to be substituted. +The name may be surrounded by curly braces +(${}) +to separate the name from surrounding characters if necessary. +Curly braces are required when you use +Python list subscripting/slicing notation on a variable +to select one or more items from a list, +or access a variable's special attributes, +or use Python expression substitution. + + + Besides regular &consvars;, scons provides the following -special variables for each command execution: +special variables for use in expanding commands: @@ -6261,45 +6278,39 @@ changed since the target was last built. -Note that the above variables are reserved -and may not be assigned to in the &consenv;. +These names are reserved +and may not be assigned to or used as &consvars;. -For example, given the &consvars; -CC='cc', -targets=['foo'] -and -sources=['foo.c', 'bar.c']: +For example, the following builder call: -action='$CC -c -o $TARGET $SOURCES' +env = Environment(CC='cc') +env.Command( + target=['foo'], + source=['foo.c', 'bar.c'], + action='@echo $CC -c -o $TARGET $SOURCES' +) -would produce the command line: +would produce the following output: cc -c -o foo foo.c bar.c -Variable names may be surrounded by curly braces -({}) -to separate the name from surrounding characters which -are not part of the name. -Within the curly braces, a variable name may use -Python list subscripting/slicing notation to select one -or more items from a list. -In the previous example, the string: + +In the previous example, a string ${SOURCES[1]} -would produce: - - -bar.c - +would expand to: bar.c. + -Additionally, a variable name may +A variable name may have the following modifiers appended within the enclosing curly braces -to access properties of the interpolated string: +to access properties of the interpolated string. +These are known as special attributes. + base - @@ -6378,23 +6389,18 @@ ${SOURCE.rsrcdir} => /usr/repository/src -Modifiers can be combined, like -${TARGET.base.windows}, +Some modifiers can be combined, like ${TARGET.srcpath.base), ${TARGET.file.suffix}, etc. -Note that curly braces braces may also be used -to enclose arbitrary Python code to be evaluated. -(In fact, this is how the above modifiers are substituted, -they are simply attributes of the Python objects -that represent &cv-TARGET;, &cv-SOURCES;, etc.) +The curly brace notation may also be used +to enclose a Python expression to be evaluated. See below -for more thorough examples of -how this can be used. +for a description. -Lastly, a variable name -may be a callable Python function +A variable name +may also be a Python function associated with a &consvar; in the environment. The function should @@ -6425,6 +6431,12 @@ def foo(target, source, env, for_signature): env=Environment(FOO=foo, BAR="$FOO baz") +As a reminder, this evaluation happens when +$BAR is actually used in a +builder action. The value of env['BAR'] +will be exactly as it was set: "$FOO baz". + + You can use this feature to pass arguments to a Python function by creating a callable class that stores one or more arguments in an object, @@ -6496,17 +6508,17 @@ echo Last build occurred . > $TARGET Python Code Substitution -Any Python code within curly braces -({}) -and introduced by the variable prefix $ -will be evaluated using the Python eval statement, -with the Python globals set to -the current environment's set of &consvars;, and the result -substituted in. +If a substitutable expression using the notation +${something} does not appear to match one of +the other substitution patterns, +it is evaluated as a Python expression. +This uses Python's eval function, +with the globals parameter set to +the current environment's set of &consvars;, +and the result substituted in. So in the following case: -env['COND'] = 0 env.Command('foo.out', 'foo.in', '''echo ${COND==1 and 'FOO' or 'BAR'} > $TARGET''') @@ -6530,7 +6542,7 @@ built, not when the SConscript is being read. So if env['COND'] is changed later in the SConscript, the final value will be used. -Here's a more interesting example. Note that all of +Here's a more complete example. Note that all of COND, FOO, and @@ -6541,17 +6553,28 @@ separated by spaces. env=Environment() -env['COND'] = 0 +env['COND'] = 1 env['FOO'] = ['foo1', 'foo2'] env['BAR'] = 'barbar' env.Command('foo.out', 'foo.in', 'echo ${COND==1 and FOO or BAR} > $TARGET') - -# Will execute this: -# echo foo1 foo2 > foo.out -SCons uses the following rules when converting &consvars; into +will execute: + +echo foo1 foo2 > foo.out + + + +In point of fact, Python expression evaluation is +how the special attributes are substituted: +they are simply attributes of the Python objects +that represent &cv-TARGET;, &cv-SOURCES;, etc., +which &SCons; passes to eval which +returns the value. + + +&SCons; uses the following rules when converting &consvars; into command lines: @@ -6589,6 +6612,16 @@ contain embedded newline characters. + + +Use of the Python eval function +is considered to have security implications, since, +depending on input sources, +arbitrary unchecked strings of code can be executed by the Python interpreter. +Although &SCons; makes use of it in a somewhat restricted context, +you should be aware of this issue when using the +${python-expression-for-subst} form. + @@ -6613,8 +6646,7 @@ a Python function that will process the Node (file) and return a list of File Nodes representing the implicit -dependencies (file names) found in the contents; -or: +dependencies (file names) found in the contents. a dictionary that maps keys -- cgit v0.12 From 93525bed88d19a00f5de400086c0046011d3b833 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Fri, 9 Oct 2020 11:56:47 -0700 Subject: mark SCons/__init__.py as NoClean() so scons -c doesn't leave the tree in a broken state --- site_scons/update_build_info.py | 1 + 1 file changed, 1 insertion(+) diff --git a/site_scons/update_build_info.py b/site_scons/update_build_info.py index b9d565a..59ddf0c 100644 --- a/site_scons/update_build_info.py +++ b/site_scons/update_build_info.py @@ -14,3 +14,4 @@ def update_init_file(env): 'import SCons.compat # noqa'], ) env.Precious(si) + env.NoClean(si) # Don't clean this file as it breaks the build. -- cgit v0.12 From ea1da16f6b1601529323943f48bf94f59eb7a785 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Fri, 9 Oct 2020 12:52:33 -0700 Subject: Add test to check if manpages make it into the package. wip --- .travis.yml | 1 + .travis/verify_packages.sh | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) create mode 100755 .travis/verify_packages.sh diff --git a/.travis.yml b/.travis.yml index 263a7ea..10bc138 100644 --- a/.travis.yml +++ b/.travis.yml @@ -92,3 +92,4 @@ jobs: - python scripts/scons.py - ls -l build/dist - python build/scons-local/scons.py --version + - ./.travis/verify_packages.sh diff --git a/.travis/verify_packages.sh b/.travis/verify_packages.sh new file mode 100755 index 0000000..6ce4e8f --- /dev/null +++ b/.travis/verify_packages.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -e +set -x + +echo "Checking wheel file" +wheel_man_files=$(unzip -l build/dist/SCons-*-py3-none-any.whl | grep -e '[a-z].1$' | tee /dev/stderr | wc -l) +echo "Number of manpage files: $wheel_man_files" + +echo "Checking tgz sdist package" +tgz_man_files=$(tar tvfz build/dist/SCons-*.tar.gz | grep -e '[a-z].1$' | tee /dev/stderr | wc -l) + +echo "Checking zip sdist package" +zip_man_files=$(unzip -l build/dist/SCons-*.zip | grep -e '[a-z].1$' | tee /dev/stderr | wc -l) + +if [[ $wheel_man_files =~ /\\s+4/ ]]; then + echo "Manpages not in wheel" + exit 1 +fi \ No newline at end of file -- cgit v0.12 From da582df6a7728a16830efd79b811ba19fc8ebb55 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Fri, 9 Oct 2020 12:52:57 -0700 Subject: scons.1 sconsign.1 scons-time.1 now make it into zip, tgz sdist and wheel --- .gitignore | 4 ++++ MANIFEST.in | 2 ++ SCons/__init__.py | 8 ++++---- SConstruct | 19 ++++++++++++++----- setup.cfg | 12 ++++++++++-- setup.py | 3 ++- site_scons/scons_local_package.py | 10 +++++++++- 7 files changed, 45 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index ec824d7..a7b59b3 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,7 @@ htmlcov # Mac junk **/.DS_Store + + +# SCons specific +*.1 diff --git a/MANIFEST.in b/MANIFEST.in index e509c38..04ec000 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,6 +3,8 @@ recursive-include SCons/Tool/docbook * # For license file include LICENSE +include scons.1 sconsign.1 scons-time.1 +recursive-include build/doc/man *.1 diff --git a/SCons/__init__.py b/SCons/__init__.py index f3d3851..fcd665e 100644 --- a/SCons/__init__.py +++ b/SCons/__init__.py @@ -1,9 +1,9 @@ -__version__="4.0.1" +__version__="4.0.1.9998" __copyright__="Copyright (c) 2001 - 2020 The SCons Foundation" __developer__="bdbaddog" -__date__="2020-07-17 01:50:03" +__date__="2020-10-09 19:00:35" __buildsys__="ProDog2020" -__revision__="c289977f8b34786ab6c334311e232886da7e8df1" -__build__="c289977f8b34786ab6c334311e232886da7e8df1" +__revision__="93525bed88d19a00f5de400086c0046011d3b833" +__build__="93525bed88d19a00f5de400086c0046011d3b833" # make sure compatibility is always in place import SCons.compat # noqa \ No newline at end of file diff --git a/SConstruct b/SConstruct index 2cd2fde..27235ad 100644 --- a/SConstruct +++ b/SConstruct @@ -12,7 +12,9 @@ copyright_years = strftime('2001 - %Y') # This gets inserted into the man pages to reflect the month of release. month_year = strftime('%B %Y') # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -36,7 +38,7 @@ month_year = strftime('%B %Y') project = 'scons' -default_version = '4.0.1' +default_version = '4.0.1.9998' copyright = "Copyright (c) %s The SCons Foundation" % copyright_years # @@ -193,14 +195,21 @@ Export('command_line', 'env', 'whereis', 'revaction') SConscript('doc/SConscript') +# Copy manpage's into base dir for inclusign in pypi packages +man_pages = env.Install("#/", Glob("#/build/doc/man/*.1")) + # Build packages for pypi -env.Command('$DISTDIR/SCons-${VERSION}-py3-none-any.whl', ['setup.cfg', 'setup.py', 'SCons/__init__.py'], +wheel = env.Command('$DISTDIR/SCons-${VERSION}-py3-none-any.whl', ['setup.cfg', 'setup.py', 'SCons/__init__.py']+man_pages, '$PYTHON setup.py bdist_wheel') -env.Command('$DISTDIR/SCons-${VERSION}.zip', ['setup.cfg', 'setup.py', 'SCons/__init__.py'], +zip_file = env.Command('$DISTDIR/SCons-${VERSION}.zip', ['setup.cfg', 'setup.py', 'SCons/__init__.py']+man_pages, '$PYTHON setup.py sdist --format=zip') -env.Command('$DISTDIR/SCons-${VERSION}.tar.gz', ['setup.cfg', 'setup.py', 'SCons/__init__.py'], +tgz_file = env.Command('$DISTDIR/SCons-${VERSION}.tar.gz', ['setup.cfg', 'setup.py', 'SCons/__init__.py']+man_pages, '$PYTHON setup.py sdist --format=gztar') +# Now set depends so the above run in a particular order +env.Depends(tgz_file, [zip_file, wheel]) +env.AddPostAction(tgz_file, Delete(man_pages)) + # TODO add auto copyright date to README.rst, LICENSE # TODO build API DOCS diff --git a/setup.cfg b/setup.cfg index a141c67..dbb316e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,6 +28,7 @@ classifiers = Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 Environment :: Console Intended Audience :: Developers License :: OSI Approved :: MIT License @@ -60,13 +61,20 @@ console_scripts = [options.package_data] -* = *.txt, *.rst +* = *.txt, *.rst, *.1 SCons.Tool.docbook = *.* + +[options.data_files] +. = build/doc/man/scons.1 + build/doc/man/scons-time.1 + build/doc/man/sconsign.1 + [sdist] dist-dir=build/dist [bdist_wheel] ; We're now py3 only ;universal=true -dist-dir=build/dist \ No newline at end of file +dist-dir=build/dist + diff --git a/setup.py b/setup.py index 8ea0246..6d52278 100644 --- a/setup.py +++ b/setup.py @@ -7,11 +7,13 @@ from setuptools.command.build_py import build_py as build_py_orig import codecs import os.path + def read(rel_path): here = os.path.abspath(os.path.dirname(__file__)) with codecs.open(os.path.join(here, rel_path), 'r') as fp: return fp.read() + def get_version(rel_path): for line in read(rel_path).splitlines(): if line.startswith('__version__'): @@ -21,7 +23,6 @@ def get_version(rel_path): raise RuntimeError("Unable to find version string.") - exclude = ['*Tests'] diff --git a/site_scons/scons_local_package.py b/site_scons/scons_local_package.py index 5e633e9..aaf058a 100644 --- a/site_scons/scons_local_package.py +++ b/site_scons/scons_local_package.py @@ -1,4 +1,6 @@ -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -60,6 +62,12 @@ def install_local_package_files(env): fn = os.path.basename(bf) all_local_installed.append(env.SCons_revision('#/build/scons-local/%s'%fn, bf)) + # Now copy manpages into scons-local package + built_manpage_files = env.Glob('build/doc/man/*.1') + for bmp in built_manpage_files: + fn = os.path.basename(str(bmp)) + all_local_installed.append(env.SCons_revision('#/build/scons-local/%s'%fn, bmp)) + rename_files = [('scons-${VERSION}.bat', 'scripts/scons.bat'), ('scons-README', 'README-local'), ('scons-LICENSE', 'LICENSE-local')] -- cgit v0.12 From 0f2df79770497bee38709cfdad51df9e29f4e1ed Mon Sep 17 00:00:00 2001 From: William Deegan Date: Fri, 9 Oct 2020 12:54:33 -0700 Subject: Add to CHANGES.txt --- CHANGES.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.txt b/CHANGES.txt index cdf57de..4d541bf 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -23,6 +23,8 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Remove pywin32 usage from SCons.Util where it was used for accessing the registry. Python native winreg library already includes this functionality. - Remove using pywin32 to retrieve peak memory usage on Win32 for `--debug=memory` + - Fix Issue #3759 - include scons.1, sconsign.1, scons-time.1 manpages in sdist and wheel packages. + From Michał Górny: - Fix dvipdf test failure due to passing incorrect flag to dvipdf. -- cgit v0.12 From 2bc4d930ebf2fbd9d724e8634368dab58f855e58 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Fri, 9 Oct 2020 12:55:17 -0700 Subject: Add to CHANGES.txt --- CHANGES.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index 4d541bf..ae754ef 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -24,7 +24,8 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER library already includes this functionality. - Remove using pywin32 to retrieve peak memory usage on Win32 for `--debug=memory` - Fix Issue #3759 - include scons.1, sconsign.1, scons-time.1 manpages in sdist and wheel packages. - + - Change SCons's build so the generated `SCons/__init__.py` is no longer removed by `scons -c` + From Michał Górny: - Fix dvipdf test failure due to passing incorrect flag to dvipdf. -- cgit v0.12 From 428382dc6ee5f7d90affa6638045f2ea20be5b4d Mon Sep 17 00:00:00 2001 From: William Deegan Date: Fri, 9 Oct 2020 13:22:10 -0700 Subject: Fix test --- .travis/verify_packages.sh | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/.travis/verify_packages.sh b/.travis/verify_packages.sh index 6ce4e8f..de64a4f 100755 --- a/.travis/verify_packages.sh +++ b/.travis/verify_packages.sh @@ -2,17 +2,31 @@ set -e set -x +retval=0 +expected_man_file_count=3 echo "Checking wheel file" -wheel_man_files=$(unzip -l build/dist/SCons-*-py3-none-any.whl | grep -e '[a-z].1$' | tee /dev/stderr | wc -l) +wheel_man_files=$(unzip -l build/dist/SCons-*-py3-none-any.whl | grep -e '[a-z].1$' | wc -l | xargs) echo "Number of manpage files: $wheel_man_files" echo "Checking tgz sdist package" -tgz_man_files=$(tar tvfz build/dist/SCons-*.tar.gz | grep -e '[a-z].1$' | tee /dev/stderr | wc -l) +tgz_man_files=$(tar tvfz build/dist/SCons-*.tar.gz | grep -e '[a-z].1$' | wc -l |xargs) echo "Checking zip sdist package" -zip_man_files=$(unzip -l build/dist/SCons-*.zip | grep -e '[a-z].1$' | tee /dev/stderr | wc -l) +zip_man_files=$(unzip -l build/dist/SCons-*.zip | grep -e '[a-z].1$' | wc -l |xargs) -if [[ $wheel_man_files =~ /\\s+4/ ]]; then +if [[ $wheel_man_files != $expected_man_file_count ]]; then echo "Manpages not in wheel" - exit 1 -fi \ No newline at end of file + retval=1 +fi + +if [[ $tgz_man_files != $expected_man_file_count ]]; then + echo "Manpages not in tgz sdist package" + retval=2 +fi + +if [[ $zip_man_files != $expected_man_file_count ]]; then + echo "Manpages not in zip sdist package" + retval=3 +fi + +exit $retval \ No newline at end of file -- cgit v0.12 From bcad36f5f9c1a82a4cd5e8856781e49f0ea7b346 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sat, 10 Oct 2020 06:32:57 -0600 Subject: [PR #3811] revert some Command syntax changes [ci skip] Signed-off-by: Mats Wichmann --- doc/man/scons.xml | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/doc/man/scons.xml b/doc/man/scons.xml index accdae7..45bc9bd 100644 --- a/doc/man/scons.xml +++ b/doc/man/scons.xml @@ -2918,22 +2918,16 @@ target file's directory. # scons will change to the "sub" subdirectory # before executing the "cp" command. -env.Command( - target='sub/dir/foo.out', - source='sub/dir/foo.in', - action="cp dir/foo.in dir/foo.out", - chdir='sub', -) +env.Command('sub/dir/foo.out', 'sub/dir/foo.in', + "cp dir/foo.in dir/foo.out", + chdir='sub') # Because chdir is not a string, scons will change to the # target's directory ("sub/dir") before executing the # "cp" command. -env.Command( - target='sub/dir/foo.out', - source='sub/dir/foo.in', - action="cp foo.in foo.out", - chdir=True, -) +env.Command('sub/dir/foo.out', 'sub/dir/foo.in', + "cp foo.in foo.out", + chdir=1) Note that &SCons; will @@ -6519,8 +6513,9 @@ and the result substituted in. So in the following case: -env.Command('foo.out', 'foo.in', - '''echo ${COND==1 and 'FOO' or 'BAR'} > $TARGET''') +env.Command( + 'foo.out', 'foo.in', "echo ${COND==1 and 'FOO' or 'BAR'} > $TARGET" +) the command executed will be either @@ -6556,8 +6551,9 @@ env=Environment() env['COND'] = 1 env['FOO'] = ['foo1', 'foo2'] env['BAR'] = 'barbar' -env.Command('foo.out', 'foo.in', - 'echo ${COND==1 and FOO or BAR} > $TARGET') +env.Command( + 'foo.out', 'foo.in', "echo ${COND==1 and FOO or BAR} > $TARGET" +) will execute: -- cgit v0.12 From 890041c42ca0a71c5bb9550a97912fc9d9007d43 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Mon, 12 Oct 2020 06:58:44 -0600 Subject: Fix/update global AddMethod Fixes #3028 Signed-off-by: Mats Wichmann --- CHANGES.txt | 2 + SCons/Environment.py | 46 +++------------- SCons/Environment.xml | 54 +++++++------------ SCons/EnvironmentTests.py | 14 ++++- SCons/Util.py | 132 ++++++++++++++++++++++++++-------------------- 5 files changed, 119 insertions(+), 129 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index ae754ef..3a1e25c 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -63,6 +63,8 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Make sure cProfile is used if profiling - SCons was expecting the Util module to monkeypatch in cProfile as profile if available, but this is no longer being done. + - Cleanup in Util.AddMethod; detect environment instances and add + them using MethodWrapper to fix #3028. MethodWrapper moves to Util. From Simon Tegelid - Fix using TEMPFILE in multiple actions in an action list. Previously a builder, or command diff --git a/SCons/Environment.py b/SCons/Environment.py index 3040427..cc62679 100644 --- a/SCons/Environment.py +++ b/SCons/Environment.py @@ -54,6 +54,7 @@ import SCons.SConsign import SCons.Subst import SCons.Tool import SCons.Util +from SCons.Util import MethodWrapper import SCons.Warnings class _Null: @@ -180,48 +181,17 @@ def _delete_duplicates(l, keep_last): # Shannon at the following page (there called the "transplant" class): # # ASPN : Python Cookbook : Dynamically added methods to a class -# http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/81732 +# https://code.activestate.com/recipes/81732/ # # We had independently been using the idiom as BuilderWrapper, but # factoring out the common parts into this base class, and making # BuilderWrapper a subclass that overrides __call__() to enforce specific # Builder calling conventions, simplified some of our higher-layer code. +# +# Note: MethodWrapper moved to SCons.Util as it was needed there +# and otherwise we had a circular import problem. -class MethodWrapper: - """ - A generic Wrapper class that associates a method (which can - actually be any callable) with an object. As part of creating this - MethodWrapper object an attribute with the specified (by default, - the name of the supplied method) is added to the underlying object. - When that new "method" is called, our __call__() method adds the - object as the first argument, simulating the Python behavior of - supplying "self" on method calls. - - We hang on to the name by which the method was added to the underlying - base class so that we can provide a method to "clone" ourselves onto - a new underlying object being copied (without which we wouldn't need - to save that info). - """ - def __init__(self, object, method, name=None): - if name is None: - name = method.__name__ - self.object = object - self.method = method - self.name = name - setattr(self.object, name, self) - - def __call__(self, *args, **kwargs): - nargs = (self.object,) + args - return self.method(*nargs, **kwargs) - - def clone(self, new_object): - """ - Returns an object that re-binds the underlying "method" to - the specified new object. - """ - return self.__class__(new_object, self.method, self.name) - -class BuilderWrapper(MethodWrapper): +class BuilderWrapper(SCons.Util.MethodWrapper): """ A MethodWrapper subclass that that associates an environment with a Builder. @@ -248,7 +218,7 @@ class BuilderWrapper(MethodWrapper): target = [target] if source is not None and not SCons.Util.is_List(source): source = [source] - return MethodWrapper.__call__(self, target, source, *args, **kw) + return super().__call__(target, source, *args, **kw) def __repr__(self): return '' % repr(self.name) @@ -2377,7 +2347,7 @@ class OverrideEnvironment(Base): # Environment they are being constructed with and so will not # have access to overrided values. So we rebuild them with the # OverrideEnvironment so they have access to overrided values. - if isinstance(attr, (MethodWrapper, BuilderWrapper)): + if isinstance(attr, MethodWrapper): return attr.clone(self) else: return attr diff --git a/SCons/Environment.xml b/SCons/Environment.xml index bb9b90e..a1fd8ec 100644 --- a/SCons/Environment.xml +++ b/SCons/Environment.xml @@ -286,31 +286,22 @@ until the Action object is actually used. -When called with the -AddMethod() -form, -adds the specified -function -to the specified -object -as the specified method -name. -When called using the -&f-env-AddMethod; form, -adds the specified -function -to the construction environment -env -as the specified method -name. -In both cases, if -name -is omitted or -None, -the name of the -specified -function -itself is used for the method name. +Adds function to an object as a method. +function will be called with an instance +object as the first argument as for other methods. +If name is given, it is used as +the name of the new method, else the name of +function is used. + + +When the global function &f-AddMethod; is called, +the object to add the method to must be passed as the first argument; +typically this will be &Environment;, +in order to create a method which applies to all &consenvs; +subsequently constructed. +When called using the &f-env-AddMethod; form, +the method is added to the specified &consenv; only. +Added methods propagate through &f-env-Clone; calls. @@ -318,22 +309,17 @@ Examples: -# Note that the first argument to the function to -# be attached as a method must be the object through -# which the method will be called; the Python -# convention is to call it 'self'. +# Function to add must accept an instance argument. +# The Python convention is to call this 'self'. def my_method(self, arg): print("my_method() got", arg) -# Use the global AddMethod() function to add a method -# to the Environment class. This +# Use the global function to add a method to the Environment class: AddMethod(Environment, my_method) env = Environment() env.my_method('arg') -# Add the function as a method, using the function -# name for the method call. -env = Environment() +# Use the optional name argument to set the name of the method: env.AddMethod(my_method, 'other_method_name') env.other_method_name('another arg') diff --git a/SCons/EnvironmentTests.py b/SCons/EnvironmentTests.py index 8d2704d..e308865 100644 --- a/SCons/EnvironmentTests.py +++ b/SCons/EnvironmentTests.py @@ -721,7 +721,7 @@ sys.exit(0) r = env4.func2() assert r == 'func2-4', r - # Test that clones don't re-bind an attribute that the user + # Test that clones don't re-bind an attribute that the user set. env1 = Environment(FOO = '1') env1.AddMethod(func2) def replace_func2(): @@ -731,6 +731,18 @@ sys.exit(0) r = env2.func2() assert r == 'replace_func2', r + # Test clone rebinding if using global AddMethod. + env1 = Environment(FOO='1') + SCons.Util.AddMethod(env1, func2) + r = env1.func2() + assert r == 'func2-1', r + r = env1.func2('-xxx') + assert r == 'func2-1-xxx', r + env2 = env1.Clone(FOO='2') + r = env2.func2() + assert r == 'func2-2', r + + def test_Override(self): """Test overriding construction variables""" env = SubstitutionEnvironment(ONE=1, TWO=2, THREE=3, FOUR=4) diff --git a/SCons/Util.py b/SCons/Util.py index 347395f..88aeaae 100644 --- a/SCons/Util.py +++ b/SCons/Util.py @@ -27,23 +27,19 @@ import os import sys import copy import re -import types import pprint import hashlib from collections import UserDict, UserList, UserString, OrderedDict from collections.abc import MappingView +from types import MethodType, FunctionType PYPY = hasattr(sys, 'pypy_translation_info') -# Below not used? -# InstanceType = types.InstanceType - -MethodType = types.MethodType -FunctionType = types.FunctionType - -def dictify(keys, values, result={}): - for k, v in zip(keys, values): - result[k] = v +# unused? +def dictify(keys, values, result=None): + if result is None: + result = {} + result.update(dict(zip(keys, values))) return result _altsep = os.altsep @@ -633,6 +629,41 @@ class Delegate: else: return self + +class MethodWrapper: + """A generic Wrapper class that associates a method with an object. + + As part of creating this MethodWrapper object an attribute with the + specified name (by default, the name of the supplied method) is added + to the underlying object. When that new "method" is called, our + __call__() method adds the object as the first argument, simulating + the Python behavior of supplying "self" on method calls. + + We hang on to the name by which the method was added to the underlying + base class so that we can provide a method to "clone" ourselves onto + a new underlying object being copied (without which we wouldn't need + to save that info). + """ + def __init__(self, object, method, name=None): + if name is None: + name = method.__name__ + self.object = object + self.method = method + self.name = name + setattr(self.object, name, self) + + def __call__(self, *args, **kwargs): + nargs = (self.object,) + args + return self.method(*nargs, **kwargs) + + def clone(self, new_object): + """ + Returns an object that re-binds the underlying "method" to + the specified new object. + """ + return self.__class__(new_object, self.method, self.name) + + # attempt to load the windows registry module: can_read_reg = 0 try: @@ -1096,7 +1127,7 @@ def adjustixes(fname, pre, suf, ensure_suffix=False): # From Tim Peters, -# http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52560 +# https://code.activestate.com/recipes/52560 # ASPN: Python Cookbook: Remove duplicates from a sequence # (Also in the printed Python Cookbook.) @@ -1170,9 +1201,8 @@ def unique(s): return u - # From Alex Martelli, -# http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52560 +# https://code.activestate.com/recipes/52560 # ASPN: Python Cookbook: Remove duplicates from a sequence # First comment, dated 2001/10/13. # (Also in the printed Python Cookbook.) @@ -1375,42 +1405,41 @@ def make_path_relative(path): return path - -# The original idea for AddMethod() and RenameFunction() come from the +# The original idea for AddMethod() came from the # following post to the ActiveState Python Cookbook: # -# ASPN: Python Cookbook : Install bound methods in an instance -# http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/223613 -# -# That code was a little fragile, though, so the following changes -# have been wrung on it: +# ASPN: Python Cookbook : Install bound methods in an instance +# https://code.activestate.com/recipes/223613 # +# Changed as follows: # * Switched the installmethod() "object" and "function" arguments, # so the order reflects that the left-hand side is the thing being # "assigned to" and the right-hand side is the value being assigned. -# -# * Changed explicit type-checking to the "try: klass = object.__class__" -# block in installmethod() below so that it still works with the -# old-style classes that SCons uses. -# -# * Replaced the by-hand creation of methods and functions with use of -# the "new" module, as alluded to in Alex Martelli's response to the -# following Cookbook post: -# -# ASPN: Python Cookbook : Dynamically added methods to a class -# http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/81732 +# * The instance/class detection is changed a bit, as it's all +# new-style classes now with Py3. +# * The by-hand construction of the function object from renamefunction() +# is not needed, the remaining bit is now used inline in AddMethod. def AddMethod(obj, function, name=None): - """ - Adds either a bound method to an instance or the function itself (or an unbound method in Python 2) to a class. - If name is ommited the name of the specified function - is used by default. + """Adds a method to an object. + + Adds `function` to `obj` if `obj` is a class object. + Adds `function` as a bound method if `obj` is an instance object. + If `obj` looks like an environment instance, use `MethodWrapper` + to add it. If `name` is supplied it is used as the name of `function`. + + Although this works for any class object, the intent as a public + API is to be used on Environment, to be able to add a method to all + construction environments; it is preferred to use env.AddMethod + to add to an individual environment. Example:: + class A: + ... a = A() def f(self, x, y): - self.z = x + y + self.z = x + y AddMethod(f, A, "add") a.add(2, 4) print(a.z) @@ -1420,32 +1449,24 @@ def AddMethod(obj, function, name=None): if name is None: name = function.__name__ else: - function = RenameFunction(function, name) + # "rename" + function = FunctionType( + function.__code__, function.__globals__, name, function.__defaults__ + ) - # Note the Python version checks - WLB - # Python 3.3 dropped the 3rd parameter from types.MethodType if hasattr(obj, '__class__') and obj.__class__ is not type: - # "obj" is an instance, so it gets a bound method. - if sys.version_info[:2] > (3, 2): - method = MethodType(function, obj) + # obj is an instance, so it gets a bound method. + if hasattr(obj, "added_methods"): + method = MethodWrapper(obj, function, name) + obj.added_methods.append(method) else: - method = MethodType(function, obj, obj.__class__) + method = MethodType(function, obj) else: - # Handle classes + # obj is a class method = function setattr(obj, name, method) -def RenameFunction(function, name): - """ - Returns a function identical to the specified function, but with - the specified name. - """ - return FunctionType(function.__code__, - function.__globals__, - name, - function.__defaults__) - if hasattr(hashlib, 'md5'): md5 = True @@ -1523,8 +1544,7 @@ def silent_intern(x): # From Dinu C. Gherman, # Python Cookbook, second edition, recipe 6.17, p. 277. -# Also: -# http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/68205 +# Also: https://code.activestate.com/recipes/68205 # ASPN: Python Cookbook: Null Object Design Pattern class Null: -- cgit v0.12 From 394faec5f14c2a933f172ce13935f3f2ad2b7101 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sun, 11 Oct 2020 07:02:52 -0600 Subject: Drop some more Py2 compat things Change exception type in a a couple of try block to what could go wrong, Py3 would not raise UniCodeDecodeError for these cases One try-import of StringIO module sconsign does not need a decode that was claimed as compat hack Remove some sys.version_info checks Use more modern way to get Python details in test frawmework AddMethod updated and RenameFunction dropped - it had become a one-liner and had no clients other than AddMethod (never exposed as public) Signed-off-by: Mats Wichmann --- CHANGES.txt | 1 + SCons/Node/FS.py | 2 +- SCons/Node/Python.py | 13 +++++-------- SCons/Script/__init__.py | 6 +----- SCons/SubstTests.py | 4 +--- SCons/Tool/clang.py | 3 +-- SCons/Tool/clangxx.py | 3 +-- SCons/Tool/textfile.py | 4 ++-- SCons/UtilTests.py | 11 +---------- SCons/Utilities/sconsign.py | 9 ++------- testing/framework/TestRuntest.py | 2 -- testing/framework/TestSCons.py | 7 ++----- 12 files changed, 18 insertions(+), 47 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index ae754ef..48ceb57 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -63,6 +63,7 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Make sure cProfile is used if profiling - SCons was expecting the Util module to monkeypatch in cProfile as profile if available, but this is no longer being done. + - Some Python 2 compat dropped From Simon Tegelid - Fix using TEMPFILE in multiple actions in an action list. Previously a builder, or command diff --git a/SCons/Node/FS.py b/SCons/Node/FS.py index 3989a64..967f007 100644 --- a/SCons/Node/FS.py +++ b/SCons/Node/FS.py @@ -434,7 +434,7 @@ class EntryProxy(SCons.Util.Proxy): # In PY3 if a class defines __eq__, then it must explicitly provide # __hash__. Since SCons.Util.Proxy provides __eq__ we need the following - # see: https://docs.python.org/3.1/reference/datamodel.html#object.__hash__ + # see: https://docs.python.org/3/reference/datamodel.html#object.__hash__ __hash__ = SCons.Util.Delegate('__hash__') def __get_abspath(self): diff --git a/SCons/Node/Python.py b/SCons/Node/Python.py index a124c03..95fd274 100644 --- a/SCons/Node/Python.py +++ b/SCons/Node/Python.py @@ -129,7 +129,7 @@ class Value(SCons.Node.Node): self.built_value = self.value return self.built_value - def get_text_contents(self): + def get_text_contents(self) -> str: """By the assumption that the node.built_value is a deterministic product of the sources, the contents of a Value are the concatenation of all the contents of its sources. As @@ -141,16 +141,13 @@ class Value(SCons.Node.Node): contents = contents + kid.get_contents().decode() return contents - def get_contents(self): - """ - Get contents for signature calculations. - :return: bytes - """ + def get_contents(self) -> bytes: + """Get contents for signature calculations.""" text_contents = self.get_text_contents() try: return text_contents.encode() - except UnicodeDecodeError: - # Already encoded as python2 str are bytes + except AttributeError: + # Should not happen, as get_text_contents returns str return text_contents def changed_since_last_build(self, target, prev_ni): diff --git a/SCons/Script/__init__.py b/SCons/Script/__init__.py index 87e6f98..dff1567 100644 --- a/SCons/Script/__init__.py +++ b/SCons/Script/__init__.py @@ -36,11 +36,7 @@ start_time = time.time() import collections import os - -try: - from StringIO import StringIO -except ImportError: - from io import StringIO +from io import StringIO import sys diff --git a/SCons/SubstTests.py b/SCons/SubstTests.py index b6d91f0..8443cad 100644 --- a/SCons/SubstTests.py +++ b/SCons/SubstTests.py @@ -541,9 +541,7 @@ class scons_subst_TestCase(SubstTestCase): scons_subst('$foo.bar.3.0', env) except SCons.Errors.UserError as e: expect = [ - # Python 2.3, 2.4 - "SyntaxError `invalid syntax (line 1)' trying to evaluate `$foo.bar.3.0'", - # Python 2.5 + # Python 2.5 and later "SyntaxError `invalid syntax (, line 1)' trying to evaluate `$foo.bar.3.0'", ] assert str(e) in expect, e diff --git a/SCons/Tool/clang.py b/SCons/Tool/clang.py index 162daad..e39c742 100644 --- a/SCons/Tool/clang.py +++ b/SCons/Tool/clang.py @@ -84,8 +84,7 @@ def generate(env): # clang -dumpversion is of no use with pipe.stdout: line = pipe.stdout.readline() - if sys.version_info[0] > 2: - line = line.decode() + line = line.decode() match = re.search(r'clang +version +([0-9]+(?:\.[0-9]+)+)', line) if match: env['CCVERSION'] = match.group(1) diff --git a/SCons/Tool/clangxx.py b/SCons/Tool/clangxx.py index b1dc6f3..736d455 100644 --- a/SCons/Tool/clangxx.py +++ b/SCons/Tool/clangxx.py @@ -92,8 +92,7 @@ def generate(env): # clang -dumpversion is of no use with pipe.stdout: line = pipe.stdout.readline() - if sys.version_info[0] > 2: - line = line.decode() + line = line.decode() match = re.search(r'clang +version +([0-9]+(?:\.[0-9]+)+)', line) if match: env['CXXVERSION'] = match.group(1) diff --git a/SCons/Tool/textfile.py b/SCons/Tool/textfile.py index 10e07f9..c1b597f 100644 --- a/SCons/Tool/textfile.py +++ b/SCons/Tool/textfile.py @@ -74,8 +74,8 @@ def _do_subst(node, subs): if 'b' in TEXTFILE_FILE_WRITE_MODE: try: contents = bytearray(contents, 'utf-8') - except UnicodeDecodeError: - # contents is already utf-8 encoded python 2 str i.e. a byte array + except TypeError: + # TODO: this should not happen, get_text_contents returns text contents = bytearray(contents) return contents diff --git a/SCons/UtilTests.py b/SCons/UtilTests.py index 50c7ed3..72bbaf3 100644 --- a/SCons/UtilTests.py +++ b/SCons/UtilTests.py @@ -193,11 +193,7 @@ class UtilTestCase(unittest.TestCase): try: node, expect, withtags = self.tree_case_1() - if sys.version_info.major < 3: - IOStream = io.BytesIO - else: - IOStream = io.StringIO - + IOStream = io.StringIO sys.stdout = IOStream() print_tree(node, get_children) actual = sys.stdout.getvalue() @@ -249,11 +245,6 @@ class UtilTestCase(unittest.TestCase): def test_is_Dict(self): assert is_Dict({}) assert is_Dict(UserDict()) - - # os.environ is not a dictionary in python 3 - if sys.version_info < (3, 0): - assert is_Dict(os.environ) - try: class mydict(dict): pass diff --git a/SCons/Utilities/sconsign.py b/SCons/Utilities/sconsign.py index 8cde51b..3288779 100644 --- a/SCons/Utilities/sconsign.py +++ b/SCons/Utilities/sconsign.py @@ -98,11 +98,6 @@ def default_mapper(entry, name): val = eval("entry." + name) except AttributeError: val = None - if sys.version_info.major >= 3 and isinstance(val, bytes): - # This is a dirty hack for py 2/3 compatibility. csig is a bytes object - # in Python3 while Python2 bytes are str. Hence, we decode the csig to a - # Python3 string - val = val.decode() return str(val) @@ -327,8 +322,8 @@ class Do_SConsignDB: sys.stderr.write("sconsign: ignoring invalid `%s' file `%s': %s\n" % (self.dbm_name, fname, e)) exc_type, _, _ = sys.exc_info() - if exc_type.__name__ == "ValueError" and sys.version_info < (3,0,0): - sys.stderr.write("Python 2 only supports pickle protocols 0-2.\n") + if exc_type.__name__ == "ValueError": + sys.stderr.write("unrecognized pickle protocol.\n") return if Print_Directories: diff --git a/testing/framework/TestRuntest.py b/testing/framework/TestRuntest.py index 801f710..a70110c 100644 --- a/testing/framework/TestRuntest.py +++ b/testing/framework/TestRuntest.py @@ -36,8 +36,6 @@ else: pythonstring = python pythonstring = pythonstring.replace('\\', '\\\\') pythonflags = '' -if sys.version_info[0] < 3: - pythonflags = ' -tt' failing_test_template = """\ import sys diff --git a/testing/framework/TestSCons.py b/testing/framework/TestSCons.py index a7eb4c5..6ec63a9 100644 --- a/testing/framework/TestSCons.py +++ b/testing/framework/TestSCons.py @@ -1491,12 +1491,9 @@ SConscript(sconscript) self.run(program=python, stdin="""\ import sysconfig, sys, os.path py_ver = 'python%d%d' % sys.version_info[:2] -# use distutils to help find include and lib path -# TODO: PY3 fine to use sysconfig.get_config_var("INCLUDEPY") try: - import distutils.sysconfig - exec_prefix = distutils.sysconfig.EXEC_PREFIX - include = distutils.sysconfig.get_python_inc() + exec_prefix = sysconfig.get_config_var("exec_prefix") + include = sysconfig.get_config_var("INCLUDEPY") print(include) lib_path = os.path.join(exec_prefix, 'libs') if not os.path.exists(lib_path): -- cgit v0.12 From 60dfff82a334b07f3f409fe78cbb8771ebf738f1 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Tue, 13 Oct 2020 07:56:28 -0600 Subject: One more "drop py2"ism from test framework Signed-off-by: Mats Wichmann --- testing/framework/TestCmd.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/framework/TestCmd.py b/testing/framework/TestCmd.py index 558bf44..76609a0 100644 --- a/testing/framework/TestCmd.py +++ b/testing/framework/TestCmd.py @@ -344,13 +344,13 @@ def is_List(e): def to_bytes(s): - if isinstance(s, bytes) or bytes is str: + if isinstance(s, bytes): return s return bytes(s, 'utf-8') def to_str(s): - if bytes is str or is_String(s): + if is_String(s): return s return str(s, 'utf-8') @@ -469,7 +469,7 @@ def match_exact(lines=None, matches=None, newline=os.sep): :returns: an object (1) on match, else None, like re.match """ - if isinstance(lines, bytes) or bytes is str: + if isinstance(lines, bytes): newline = to_bytes(newline) if not is_List(lines): -- cgit v0.12 From 01ac999898b9cc01b0871fbf0f52d6593abf5226 Mon Sep 17 00:00:00 2001 From: Adam Gross Date: Tue, 13 Oct 2020 15:13:58 -0400 Subject: Fix occasional test failures when running multiple jobs The way runtest.py passes the list of fixture directories is racy because it sets it in os.environ['FIXTURE_DIRS'] and then spawns the subprocess, counting on Python to start the subprocess before that list is overwritten when spawning the next directory. At least on Windows, the environment is not copied in subprocess.run so runtest.py may overwrite the list of fixture directories with the list for test #2 while the subprocess module is still kicking off test #1. I was able to easily reproduce this by running the command: `python runtest.py -j 2 test\MSVC\VSWHERE.py test\AS\ASPPFLAGS.py` a few times in a row. However, with this fix, that command repeatedly succeeds. To validate ths fix, I also ran that command with "--xml a.xml" and "--xml a.xml --nopipefiles" to validate that those other executors worked correctly. --- CHANGES.txt | 2 ++ runtest.py | 37 +++++++++++++++++++++++-------------- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index ae754ef..07aa4ca 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -33,6 +33,8 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER From Adam Gross: - Fix minor bug affecting SCons.Node.FS.File.get_csig()'s usage of the MD5 chunksize. User-facing behavior does not change with this fix (GH Issue #3726). + - Fix occasional test failures caused by not being able to find a file or directory fixture + when running multiple tests with multiple jobs. From Joachim Kuebart: - Suppress missing SConscript deprecation warning if `must_exist=False` diff --git a/runtest.py b/runtest.py index 16c4f0f..f751212 100755 --- a/runtest.py +++ b/runtest.py @@ -293,8 +293,8 @@ def escape(s): if not catch_output: # Without any output suppressed, we let the subprocess # write its stuff freely to stdout/stderr. - def spawn_it(command_args): - cp = subprocess.run(command_args, shell=False) + def spawn_it(command_args, env): + cp = subprocess.run(command_args, shell=False, env=env) return cp.stdout, cp.stderr, cp.returncode else: # Else, we catch the output of both pipes... @@ -310,7 +310,7 @@ else: # http://http://thraxil.org/users/anders/posts/2008/03/13/Subprocess-Hanging-PIPE-is-your-enemy/ # and pass temp file objects to Popen() instead of the ubiquitous # subprocess.PIPE. - def spawn_it(command_args): + def spawn_it(command_args, env): # Create temporary files tmp_stdout = tempfile.TemporaryFile(mode='w+t') tmp_stderr = tempfile.TemporaryFile(mode='w+t') @@ -318,7 +318,8 @@ else: cp = subprocess.run(command_args, stdout=tmp_stdout, stderr=tmp_stderr, - shell=False) + shell=False, + env=env) try: # Rewind to start of files @@ -348,11 +349,12 @@ else: # (but the subprocess isn't writing anything there). # Hence a deadlock. # Be dragons here! Better don't use this! - def spawn_it(command_args): + def spawn_it(command_args, env): cp = subprocess.run(command_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - shell=False) + shell=False, + env=env) return cp.stdout, cp.stderr, cp.returncode @@ -380,8 +382,8 @@ class RuntestBase(ABC): class SystemExecutor(RuntestBase): """ Test class for tests executed with spawn_it() """ - def execute(self): - self.stderr, self.stdout, s = spawn_it(self.command_args) + def execute(self, env): + self.stderr, self.stdout, s = spawn_it(self.command_args, env) self.status = s if s < 0 or s > 2: sys.stdout.write("Unexpected exit status %d\n" % s) @@ -400,7 +402,7 @@ class PopenExecutor(RuntestBase): # and the 'allow_pipe_files' option, please check out the # definition of spawn_it() above. if allow_pipe_files: - def execute(self): + def execute(self, env): # Create temporary files tmp_stdout = tempfile.TemporaryFile(mode='w+t') tmp_stderr = tempfile.TemporaryFile(mode='w+t') @@ -408,7 +410,8 @@ class PopenExecutor(RuntestBase): cp = subprocess.run(self.command_str.split(), stdout=tmp_stdout, stderr=tmp_stderr, - shell=False) + shell=False, + env=env) self.status = cp.returncode try: @@ -423,11 +426,12 @@ class PopenExecutor(RuntestBase): tmp_stdout.close() tmp_stderr.close() else: - def execute(self): + def execute(self, env): cp = subprocess.run(self.command_str.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, - shell=False) + shell=False, + env=env) self.status = cp.returncode self.stdout = cp.stdout self.stderr = cp.stderr @@ -756,11 +760,16 @@ def run_test(t, io_lock=None, run_async=True): if head: fixture_dirs.append(head) fixture_dirs.append(os.path.join(scriptpath, 'test', 'fixture')) - os.environ['FIXTURE_DIRS'] = os.pathsep.join(fixture_dirs) + + # Set the list of fixture dirs directly in the environment. Just putting + # it in os.environ and spawning the process is racy. Make it reliable by + # overriding the environment passed to execute(). + env = dict(os.environ) + env['FIXTURE_DIRS'] = os.pathsep.join(fixture_dirs) test_start_time = time_func() if execute_tests: - t.execute() + t.execute(env) t.test_time = time_func() - test_start_time log_result(t, io_lock=io_lock) -- cgit v0.12 From 2cdc9dd4c9503c0f9e15c0cf636661fc99c9eeca Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Tue, 13 Oct 2020 17:48:35 -0600 Subject: [PR #3815] change per reveiw: drop unneeded try block in Python node Signed-off-by: Mats Wichmann --- CHANGES.txt | 2 +- SCons/Node/Python.py | 9 ++------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 48ceb57..8ba4091 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -63,7 +63,7 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Make sure cProfile is used if profiling - SCons was expecting the Util module to monkeypatch in cProfile as profile if available, but this is no longer being done. - - Some Python 2 compat dropped + - Some Python 2 compatibility code dropped From Simon Tegelid - Fix using TEMPFILE in multiple actions in an action list. Previously a builder, or command diff --git a/SCons/Node/Python.py b/SCons/Node/Python.py index 95fd274..0eab05c 100644 --- a/SCons/Node/Python.py +++ b/SCons/Node/Python.py @@ -143,19 +143,14 @@ class Value(SCons.Node.Node): def get_contents(self) -> bytes: """Get contents for signature calculations.""" - text_contents = self.get_text_contents() - try: - return text_contents.encode() - except AttributeError: - # Should not happen, as get_text_contents returns str - return text_contents + return self.get_text_contents().encode() def changed_since_last_build(self, target, prev_ni): cur_csig = self.get_csig() try: return cur_csig != prev_ni.csig except AttributeError: - return 1 + return True def get_csig(self, calc=None): """Because we're a Python value node and don't have a real -- cgit v0.12 From 7577f9454ceabb40b961301e062f3c197173f8ca Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Wed, 14 Oct 2020 21:02:00 -0600 Subject: Update CHANGES.txt for AddMethod changes. [ci skip] Signed-off-by: Mats Wichmann --- CHANGES.txt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 3a1e25c..602e689 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -63,8 +63,11 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Make sure cProfile is used if profiling - SCons was expecting the Util module to monkeypatch in cProfile as profile if available, but this is no longer being done. - - Cleanup in Util.AddMethod; detect environment instances and add - them using MethodWrapper to fix #3028. MethodWrapper moves to Util. + - Cleanup in SCons.Util.AddMethod. If called with an environment instance + as the object to modify, the method would not be correctly set up in + any Clone of that instance. Now tries to detect this and calls + MethodWrapper to set up the method the same way env.AddMethod does. + MethodWrapper moved to Util to avoid a circular import. Fixes #3028. From Simon Tegelid - Fix using TEMPFILE in multiple actions in an action list. Previously a builder, or command -- cgit v0.12 From 948e8afafe3a0b4c3774aca6a935b20972a9e762 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Fri, 16 Oct 2020 11:15:33 -0700 Subject: Add custom up_to_date for TimeSCons to handle individual SConscript timing output during null build run --- testing/framework/TestSCons.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/testing/framework/TestSCons.py b/testing/framework/TestSCons.py index 6ec63a9..36b9df4 100644 --- a/testing/framework/TestSCons.py +++ b/testing/framework/TestSCons.py @@ -1823,7 +1823,11 @@ class TimeSCons(TestSCons): # TODO(sgk): allow the caller to specify the target (argument) # that must be up-to-date. self.add_timing_options(kw) - self.up_to_date(arguments='.', **kw) + + # Build up regex for + # SConscript:/private/var/folders/ng/48pttrpj239fw5rmm3x65pxr0000gn/T/testcmd.12081.pk1bv5i5/SConstruct took 533.646 ms + read_str = 'SConscript:.*\n' + self.up_to_date(arguments='.', read_str=read_str, **kw) sys.stdout.write(self.stdout()) stats = self.collect_stats(self.stdout()) # time-commands should always be 0.0 on a null build, because @@ -1885,6 +1889,27 @@ class TimeSCons(TestSCons): destination = source.replace(source_dir, dest_dir) shutil.copy2(source, destination) + def up_to_date(self, arguments='.', read_str="", **kw): + """Asserts that all of the targets listed in arguments is + up to date, but does not make any assumptions on other targets. + This function is most useful in conjunction with the -n option. + Note: This custom version for timings tests does NOT escape + read_str. + """ + s = "" + for arg in arguments.split(): + s = s + "scons: `%s' is up to date.\n" % arg + kw['arguments'] = arguments + stdout = self.wrap_stdout(read_str="REPLACEME", build_str=s) + # Append '.*' so that timing output that comes after the + # up-to-date output is okay. + stdout = re.escape(stdout) + '.*' + stdout = stdout.replace('REPLACEME', read_str) + kw['stdout'] = stdout + kw['match'] = self.match_re_dotall + self.run(**kw) + + # In some environments, $AR will generate a warning message to stderr # if the library doesn't previously exist and is being created. One -- cgit v0.12 From dabd7986f70ef050228d1ee8e4ace67419d7efc0 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Mon, 26 Oct 2020 14:56:03 -0600 Subject: Fix a sider complaint in EnvironmentTests.py Signed-off-by: Mats Wichmann --- SCons/EnvironmentTests.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/SCons/EnvironmentTests.py b/SCons/EnvironmentTests.py index e308865..53dd9a7 100644 --- a/SCons/EnvironmentTests.py +++ b/SCons/EnvironmentTests.py @@ -33,7 +33,13 @@ from collections import UserDict as UD, UserList as UL import TestCmd import TestUnit -from SCons.Environment import * +from SCons.Environment import ( + Environment, + NoSubstitutionProxy, + OverrideEnvironment, + SubstitutionEnvironment, + is_valid_construction_var, +) import SCons.Warnings def diff_env(env1, env2): -- cgit v0.12