From e97b156243b38ea86812bc8986f3ef8a639aa3eb Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Fri, 6 Aug 2021 10:44:37 -0500 Subject: Expanded ninja Mkdir to also support Mkdir actions. --- CHANGES.txt | 1 + RELEASE.txt | 3 +- SCons/Tool/ninja/NinjaState.py | 1 + SCons/Tool/ninja/Rules.py | 8 +-- SCons/Tool/ninja/__init__.py | 2 +- test/ninja/mkdir_function_command.py | 121 +++++++++++++++++++++++++++++++++++ 6 files changed, 129 insertions(+), 7 deletions(-) create mode 100644 test/ninja/mkdir_function_command.py diff --git a/CHANGES.txt b/CHANGES.txt index 6ac5519..1d8f9c6 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -16,6 +16,7 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER From Daniel Moody: - Fix ninja tool to never use for_sig substitution because ninja does not use signatures. This issue affected CommandGeneratorAction function actions specifically. + - Expanded ninja Mkdir to also support Mkdir actions. From Mats Wichmann: - Two small Python 3.10 fixes: one more docstring turned into raw diff --git a/RELEASE.txt b/RELEASE.txt index faa017f..3f9bd5a 100755 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -19,8 +19,7 @@ DEPRECATED FUNCTIONALITY CHANGED/ENHANCED EXISTING FUNCTIONALITY --------------------------------------- -- List modifications to existing features, where the previous behavior - wouldn't actually be considered a bug +- Expanded ninja Mkdir to also support Mkdir actions. FIXES ----- diff --git a/SCons/Tool/ninja/NinjaState.py b/SCons/Tool/ninja/NinjaState.py index e7d9882..f1aa7c6 100644 --- a/SCons/Tool/ninja/NinjaState.py +++ b/SCons/Tool/ninja/NinjaState.py @@ -536,6 +536,7 @@ class SConsToNinjaTranslator: # TODO: use command action #3573 "installFunc": _install_action_function, "MkdirFunc": _mkdir_action_function, + "Mkdir": _mkdir_action_function, "LibSymlinksActionFunction": _lib_symlink_action_function, "Copy": _copy_action_function } diff --git a/SCons/Tool/ninja/Rules.py b/SCons/Tool/ninja/Rules.py index a2f6bc5..e7d19e3 100644 --- a/SCons/Tool/ninja/Rules.py +++ b/SCons/Tool/ninja/Rules.py @@ -37,7 +37,7 @@ def _install_action_function(_env, node): def _mkdir_action_function(env, node): return { "outputs": get_outputs(node), - "rule": get_rule(node, "CMD"), + "rule": get_rule(node, "GENERATED_CMD"), # implicit explicitly omitted, we translate these so they can be # used by anything that depends on these but commonly this is # hit with a node that will depend on all of the fake @@ -45,8 +45,8 @@ def _mkdir_action_function(env, node): # to an invalid ninja file. "variables": { # On Windows mkdir "-p" is always on - "cmd": "{mkdir}".format( - mkdir="mkdir $out & exit 0" if env["PLATFORM"] == "win32" else "mkdir -p $out", + "cmd": "mkdir {args}".format( + args = ' '.join(get_outputs(node)) + " & exit /b 0" if env["PLATFORM"] == "win32" else "-p " + ' '.join(get_outputs(node)), ), }, } @@ -58,7 +58,7 @@ def _copy_action_function(env, node): "inputs": get_inputs(node), "rule": get_rule(node, "CMD"), "variables": { - "cmd": "$COPY $in $out", + "cmd": "$COPY", }, } diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index 363c05d..12edff8 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -186,7 +186,7 @@ def generate(env): env.Append(BUILDERS={"Ninja": ninja_builder_obj}) env["NINJA_ALIAS_NAME"] = env.get("NINJA_ALIAS_NAME", "generate-ninja") - env['NINJA_DIR'] = env.get("NINJA_DIR", env.Dir(".ninja").path) + env['NINJA_DIR'] = env.get("NINJA_DIR", ".ninja") # here we allow multiple environments to construct rules and builds # into the same ninja file diff --git a/test/ninja/mkdir_function_command.py b/test/ninja/mkdir_function_command.py new file mode 100644 index 0000000..247181a --- /dev/null +++ b/test/ninja/mkdir_function_command.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +import os + +import TestSCons +from TestCmd import IS_WINDOWS + +test = TestSCons.TestSCons() + +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + +_python_ = TestSCons._python_ +_exe = TestSCons._exe + +ninja_bin = os.path.abspath(os.path.join( + ninja.__file__, + os.pardir, + 'data', + 'bin', + 'ninja' + _exe)) + +test.write('SConstruct', """ +SetOption('experimental','ninja') +DefaultEnvironment(tools=[]) +env = Environment() +env.Tool('ninja') +env.Dir('test0') +env.Dir('test1/test2') +env.Dir('test3/test4') +env.Command('test5', 'test3/test4', Mkdir('$TARGET')) +env.Command('test6/test7', ['test1/test2', 'test0'], Mkdir('$TARGET')) +""") + +# generate simple build +test.run(stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' % locals()) +test.must_exist([ + test.workpath('test0'), + test.workpath('test1/test2'), + test.workpath('test3/test4'), + test.workpath('test5'), + test.workpath('test6/test7') +]) + +# clean build and ninja files +test.run(arguments='-c', stdout=None) +test.must_contain_all_lines(test.stdout(), [ + 'Removed build.ninja']) +os.rmdir('test0') +os.rmdir('test1/test2') +os.rmdir('test1') +os.rmdir('test3/test4') +os.rmdir('test3') +os.rmdir('test5') +os.rmdir('test6/test7') +os.rmdir('test6') + +test.must_not_exist([ + test.workpath('test0'), + test.workpath('test1/test2'), + test.workpath('test3/test4'), + test.workpath('test5'), + test.workpath('test6/test7') +]) + +# only generate the ninja file +test.run(arguments='--disable-execute-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_not_exist([ + test.workpath('test0'), + test.workpath('test1/test2'), + test.workpath('test3/test4'), + test.workpath('test5'), + test.workpath('test6/test7') +]) + +# run ninja independently +program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin +test.run(program=program, stdout=None) +test.must_exist([ + test.workpath('test0'), + test.workpath('test1/test2'), + test.workpath('test3/test4'), + test.workpath('test5'), + test.workpath('test6/test7') +]) + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: -- cgit v0.12 From 3e172324f2e47581c173e1bd0ae8d8d1237067ae Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Tue, 7 Sep 2021 10:20:59 -0500 Subject: fix test to ensure no callbacks for mkdir actions --- test/ninja/mkdir_function_command.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/ninja/mkdir_function_command.py b/test/ninja/mkdir_function_command.py index 247181a..8a17623 100644 --- a/test/ninja/mkdir_function_command.py +++ b/test/ninja/mkdir_function_command.py @@ -60,6 +60,7 @@ env.Command('test6/test7', ['test1/test2', 'test0'], Mkdir('$TARGET')) test.run(stdout=None) test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_single_instance_of(test.stdout(), ["scons: Building targets ..."]) test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' % locals()) test.must_exist([ test.workpath('test0'), @@ -73,12 +74,12 @@ test.must_exist([ test.run(arguments='-c', stdout=None) test.must_contain_all_lines(test.stdout(), [ 'Removed build.ninja']) -os.rmdir('test0') +os.rmdir('test0') os.rmdir('test1/test2') os.rmdir('test1') os.rmdir('test3/test4') os.rmdir('test3') -os.rmdir('test5') +os.rmdir('test5') os.rmdir('test6/test7') os.rmdir('test6') @@ -104,6 +105,7 @@ test.must_not_exist([ # run ninja independently program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin test.run(program=program, stdout=None) +test.must_not_contain_any_line(test.stdout(), ["scons: Building targets ..."]) test.must_exist([ test.workpath('test0'), test.workpath('test1/test2'), -- cgit v0.12 From 9b420fed7e736c75cc5fd81613355f0b3db6e873 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Thu, 5 Aug 2021 15:38:22 -0500 Subject: update ninja tool to support $ in scons command line args from https://github.com/mongodb/mongo/commit/dee0f733cdb3cab7714326574efc8a1ae4ec750f --- CHANGES.txt | 7 ++ RELEASE.txt | 6 ++ SCons/Tool/ninja/NinjaState.py | 12 ++-- SCons/Tool/ninja/__init__.py | 2 +- test/ninja/ninja_command_line.py | 82 ++++++++++++++++++++++ .../sconstruct_ninja_command_line | 12 ++++ 6 files changed, 114 insertions(+), 7 deletions(-) create mode 100644 test/ninja/ninja_command_line.py create mode 100644 test/ninja/ninja_test_sconscripts/sconstruct_ninja_command_line diff --git a/CHANGES.txt b/CHANGES.txt index bedc08d..a27e0af 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -34,6 +34,13 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Fix ninja tool to never use for_sig substitution because ninja does not use signatures. This issue affected CommandGeneratorAction function actions specifically. - Added support for the PCH environment variable to support subst generators. + - Fix command line escaping for ninja dollar sign escape. Without escaping ninja properly, + the ninja file scons regenerate and callback invocations will lose the $ characters used in + the scons command line which ninja uses itself for escaping. For Example: + scons BUILD=xyz OTHERVAR=$BUILD + Prior to this fix, it would cause ninja to fail to escape the dollar sign, leading to the + single dollar sign being used as a ninja escape character in the ninja file. + From Mats Wichmann: - Two small Python 3.10 fixes: one more docstring turned into raw diff --git a/RELEASE.txt b/RELEASE.txt index f1411e4..439d7c4 100755 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -38,6 +38,12 @@ FIXES - Fix PCH not being evaluated by subst() where necessary. - Fix issue #4021. Change the way subst() is used in Textfile() to not evaluate '$$(' -> '$', but instead it should yield '$('. +- Fix command line escaping for ninja dollar sign escape. Without escaping ninja properly, + the ninja file scons regenerate and callback invocations will lose the $ characters used in + the scons command line which ninja uses itself for escaping. For Example: + scons BUILD=xyz OTHERVAR=$BUILD + Prior to this fix, it would cause ninja to fail to escape the dollar sign, leading to the + single dollar sign being used as a ninja escape character in the ninja file. IMPROVEMENTS ------------ diff --git a/SCons/Tool/ninja/NinjaState.py b/SCons/Tool/ninja/NinjaState.py index e7d9882..9c988b9 100644 --- a/SCons/Tool/ninja/NinjaState.py +++ b/SCons/Tool/ninja/NinjaState.py @@ -44,7 +44,7 @@ from .Methods import get_command class NinjaState: """Maintains state of Ninja build system as it's translated from SCons.""" - def __init__(self, env, ninja_file, writer_class): + def __init__(self, env, ninja_file, ninja_syntax): self.env = env self.ninja_file = ninja_file @@ -63,7 +63,7 @@ class NinjaState: # its in the path later self.ninja_bin_path = ninja_bin - self.writer_class = writer_class + self.writer_class = ninja_syntax.Writer self.__generated = False self.translator = SConsToNinjaTranslator(env) self.generated_suffixes = env.get("NINJA_GENERATED_SOURCE_SUFFIXES", []) @@ -80,23 +80,23 @@ class NinjaState: # shell quoting on whatever platform it's run on. Here we use it # to make the SCONS_INVOCATION variable properly quoted for things # like CCFLAGS - escape = env.get("ESCAPE", lambda x: x) + scons_escape = env.get("ESCAPE", lambda x: x) # if SCons was invoked from python, we expect the first arg to be the scons.py # script, otherwise scons was invoked from the scons script python_bin = '' if os.path.basename(sys.argv[0]) == 'scons.py': - python_bin = escape(sys.executable) + python_bin = ninja_syntax.escape(scons_escape(sys.executable)) self.variables = { "COPY": "cmd.exe /c 1>NUL copy" if sys.platform == "win32" else "cp", "SCONS_INVOCATION": '{} {} --disable-ninja __NINJA_NO=1 $out'.format( python_bin, " ".join( - [escape(arg) for arg in sys.argv if arg not in COMMAND_LINE_TARGETS] + [ninja_syntax.escape(scons_escape(arg)) for arg in sys.argv if arg not in COMMAND_LINE_TARGETS] ), ), "SCONS_INVOCATION_W_TARGETS": "{} {}".format( - python_bin, " ".join([escape(arg) for arg in sys.argv]) + python_bin, " ".join([ninja_syntax.escape(scons_escape(arg)) for arg in sys.argv]) ), # This must be set to a global default per: # https://ninja-build.org/manual.html#_deps diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index 363c05d..6c70952 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -383,7 +383,7 @@ def generate(env): ninja_syntax = importlib.import_module(".ninja_syntax", package='ninja') if NINJA_STATE is None: - NINJA_STATE = NinjaState(env, ninja_file[0], ninja_syntax.Writer) + NINJA_STATE = NinjaState(env, ninja_file[0], ninja_syntax) # TODO: this is hacking into scons, preferable if there were a less intrusive way # We will subvert the normal builder execute to make sure all the ninja file is dependent diff --git a/test/ninja/ninja_command_line.py b/test/ninja/ninja_command_line.py new file mode 100644 index 0000000..d8e3c08 --- /dev/null +++ b/test/ninja/ninja_command_line.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +import os + +import TestSCons +from TestCmd import IS_WINDOWS + +test = TestSCons.TestSCons() + +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + +_python_ = TestSCons._python_ +_exe = TestSCons._exe + +ninja_bin = os.path.abspath(os.path.join( + ninja.__file__, + os.pardir, + 'data', + 'bin', + 'ninja' + _exe)) + +test.dir_fixture('ninja-fixture', 'src') + +test.file_fixture('ninja_test_sconscripts/sconstruct_ninja_command_line', 'SConstruct') + +# generate simple build +test.run(arguments=['BUILD=build', 'OTHER_VAR=$BUILD'], stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' % locals()) +test.run(program=test.workpath('foo' + _exe), stdout="foo.c") + +# clean build and ninja files +test.run(arguments=['-c', 'BUILD=build', 'OTHER_VAR=$BUILD'], stdout=None) +test.must_contain_all_lines(test.stdout(), [ + 'Removed build%sbar2.c' % os.sep, + 'Removed build%sbar2.o' % os.sep, + 'Removed foo', + 'Removed build.ninja']) + +# only generate the ninja file +test.run(arguments=['--disable-execute-ninja', 'BUILD=build', 'OTHER_VAR=$BUILD'], stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_not_exist(test.workpath('foo' + _exe)) + +# run ninja independently +program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin +test.run(program=program, stdout=None) +test.run(program=test.workpath('foo' + _exe), stdout="foo.c") + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/ninja/ninja_test_sconscripts/sconstruct_ninja_command_line b/test/ninja/ninja_test_sconscripts/sconstruct_ninja_command_line new file mode 100644 index 0000000..14ae633 --- /dev/null +++ b/test/ninja/ninja_test_sconscripts/sconstruct_ninja_command_line @@ -0,0 +1,12 @@ +SetOption('experimental','ninja') +DefaultEnvironment(tools=[]) + +env_vars=Variables(args=ARGUMENTS) +env_vars.Add('BUILD') +env_vars.Add('OTHER_VAR') +env = Environment(variables=env_vars) + +env.VariantDir(env['OTHER_VAR'], 'src') +env.Tool('ninja') +env.Textfile('$BUILD/bar2.c', open('src/foo.c').read()) +env.Program(target = 'foo', source = '$BUILD/bar2.c', CPPPATH='src') -- cgit v0.12 From 1b357561a89f090a7e5a9ee6b9dc5e4cac9e5c1e Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sun, 26 Sep 2021 13:33:57 -0700 Subject: minor changes to RELEASE.txt and CHANGES.txt to have explicit example --- CHANGES.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index a27e0af..8948208 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -41,7 +41,6 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER Prior to this fix, it would cause ninja to fail to escape the dollar sign, leading to the single dollar sign being used as a ninja escape character in the ninja file. - From Mats Wichmann: - Two small Python 3.10 fixes: one more docstring turned into raw because it contained an escape; updated "helpful" syntax error message -- cgit v0.12 From 82eb7927b8edec0e680728e65419e6c24667250f Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Tue, 5 Oct 2021 15:24:59 -0500 Subject: Updated ninja tool to allow for NINJA_FORCE_SCONS_BUILD to be used --- CHANGES.txt | 3 + RELEASE.txt | 2 +- SCons/Tool/ninja/Methods.py | 10 ++- SCons/Tool/ninja/NinjaState.py | 15 +++- SCons/Tool/ninja/__init__.py | 2 +- SCons/Tool/ninja/ninja.xml | 11 +++ test/ninja/force_scons_callback.py | 89 ++++++++++++++++++++++ .../sconstruct_force_scons_callback | 8 ++ 8 files changed, 136 insertions(+), 4 deletions(-) create mode 100644 test/ninja/force_scons_callback.py create mode 100644 test/ninja/ninja_test_sconscripts/sconstruct_force_scons_callback diff --git a/CHANGES.txt b/CHANGES.txt index 3e26ade..59b7621 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -36,6 +36,9 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER issue affected CommandGeneratorAction function actions specifically. - Added support for the PCH environment variable to support subst generators. + From Daniel Moody: + - Added ninja API 'NINJA_FORCE_SCONS_BUILD' to force a node to callback to scons. + From Mats Wichmann: - Two small Python 3.10 fixes: one more docstring turned into raw because it contained an escape; updated "helpful" syntax error message diff --git a/RELEASE.txt b/RELEASE.txt index f1411e4..437d061 100755 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -9,7 +9,7 @@ Here is a summary of the changes since 4.2.0: NEW FUNCTIONALITY ----------------- -- List new features (presumably why a checkpoint is being released) +- Added ninja API 'NINJA_FORCE_SCONS_BUILD' to force a node to callback to scons. DEPRECATED FUNCTIONALITY ------------------------ diff --git a/SCons/Tool/ninja/Methods.py b/SCons/Tool/ninja/Methods.py index 758b4fb..073cf71 100644 --- a/SCons/Tool/ninja/Methods.py +++ b/SCons/Tool/ninja/Methods.py @@ -140,6 +140,8 @@ def get_command(env, node, action): # pylint: disable=too-many-branches provider = __NINJA_RULE_MAPPING.get(comstr, get_generic_shell_command) rule, variables, provider_deps = provider(sub_env, node, action, tlist, slist, executor=executor) + if node.get_env().get('NINJA_FORCE_SCONS_BUILD'): + rule = 'TEMPLATE' # Get the dependencies for all targets implicit = list({dep for tgt in tlist for dep in get_dependencies(tgt)}) @@ -264,6 +266,12 @@ def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False, custom variables["env"] += env.subst( "export %s=%s;" % (key, value), target=targets, source=sources, executor=executor ) + " " - return rule, variables, [tool_command] + + if node.get_env().get('NINJA_FORCE_SCONS_BUILD'): + ret_rule = 'TEMPLATE' + else: + ret_rule = rule + + return ret_rule, variables, [tool_command] return get_response_file_command diff --git a/SCons/Tool/ninja/NinjaState.py b/SCons/Tool/ninja/NinjaState.py index e7d9882..d7b9b88 100644 --- a/SCons/Tool/ninja/NinjaState.py +++ b/SCons/Tool/ninja/NinjaState.py @@ -299,7 +299,7 @@ class NinjaState: ninja.comment("Generated by scons. DO NOT EDIT.") - ninja.variable("builddir", get_path(self.env['NINJA_DIR'])) + ninja.variable("builddir", get_path(self.env.Dir(self.env['NINJA_DIR']).path)) for pool_name, size in self.pools.items(): ninja.pool(pool_name, min(self.env.get('NINJA_MAX_JOBS', size), size)) @@ -447,6 +447,19 @@ class NinjaState: template_builds.update(template_builder) if template_builds.get("outputs", []): + + # Try to clean up any dependency cycles. If we are passing an + # ouptut node to SCons, it will build any dependencys if ninja + # has not already. + for output in template_builds.get("outputs", []): + inputs = template_builds.get('inputs') + if inputs and output in inputs: + inputs.remove(output) + + implicits = template_builds.get('implicit') + if implicits and output in implicits: + implicits.remove(output) + ninja.build(**template_builds) # We have to glob the SCons files here to teach the ninja file diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index 363c05d..079de09 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -186,7 +186,7 @@ def generate(env): env.Append(BUILDERS={"Ninja": ninja_builder_obj}) env["NINJA_ALIAS_NAME"] = env.get("NINJA_ALIAS_NAME", "generate-ninja") - env['NINJA_DIR'] = env.get("NINJA_DIR", env.Dir(".ninja").path) + env['NINJA_DIR'] = env.get("NINJA_DIR", env.Dir('#/.ninja').path) # here we allow multiple environments to construct rules and builds # into the same ninja file diff --git a/SCons/Tool/ninja/ninja.xml b/SCons/Tool/ninja/ninja.xml index e87addc..c5ee15b 100644 --- a/SCons/Tool/ninja/ninja.xml +++ b/SCons/Tool/ninja/ninja.xml @@ -69,6 +69,7 @@ See its __doc__ string for a discussion of the format. NINJA_POOL NINJA_REGENERATE_DEPS NINJA_SYNTAX + NINJA_FORCE_SCONS_BUILD _NINJA_REGENERATE_DEPS_FUNC __NINJA_NO IMPLICIT_COMMAND_DEPENDENCIES @@ -315,6 +316,16 @@ See its __doc__ string for a discussion of the format. + + + + When NINJA_FORCE_SCONS_BUILD is True, this will cause the build nodes to callback to scons instead of using + ninja to build them. This is intended to be passed to the environment on the builder invocation. + It is useful if you have a build node which does something which is not easily translated into ninja. + + + + diff --git a/test/ninja/force_scons_callback.py b/test/ninja/force_scons_callback.py new file mode 100644 index 0000000..44cb7ec --- /dev/null +++ b/test/ninja/force_scons_callback.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +import os +import sys + +import TestSCons +from TestCmd import IS_WINDOWS + +test = TestSCons.TestSCons() + +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + +_python_ = TestSCons._python_ +_exe = TestSCons._exe + +ninja_bin = os.path.abspath(os.path.join( + ninja.__file__, + os.pardir, + 'data', + 'bin', + 'ninja' + _exe)) + +test.dir_fixture('ninja-fixture') + +test.file_fixture('ninja_test_sconscripts/sconstruct_force_scons_callback', 'SConstruct') + +# generate simple build +test.run(stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' % locals()) +if test.stdout().count('scons: Building targets') != 2: + test.fail_test() +test.must_match('out.txt', 'foo.c' + os.linesep) +test.must_match('out2.txt', "test2.cpp" + os.linesep) + +# clean build and ninja files +test.run(arguments='-c', stdout=None) +test.must_contain_all_lines(test.stdout(), [ + 'Removed out.txt', + 'Removed out2.txt', + 'Removed build.ninja']) + +# only generate the ninja file +test.run(arguments='--disable-execute-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_not_exist(test.workpath('out.txt')) +test.must_not_exist(test.workpath('out2.txt')) + +# run ninja independently +program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin +test.run(program=program, stdout=None) +if test.stdout().count('scons: Building targets') != 1: + test.fail_test() +test.must_match('out.txt', 'foo.c' + os.linesep) +test.must_match('out2.txt', "test2.cpp" + os.linesep) + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/ninja/ninja_test_sconscripts/sconstruct_force_scons_callback b/test/ninja/ninja_test_sconscripts/sconstruct_force_scons_callback new file mode 100644 index 0000000..55729a6 --- /dev/null +++ b/test/ninja/ninja_test_sconscripts/sconstruct_force_scons_callback @@ -0,0 +1,8 @@ +SetOption('experimental','ninja') +DefaultEnvironment(tools=[]) + +env = Environment(tools=[]) +env.Tool('ninja') + +env.Command('out.txt', 'foo.c', 'echo $SOURCE> $TARGET', NINJA_FORCE_SCONS_BUILD=True) +env.Command('out2.txt', 'test2.cpp', 'echo $SOURCE> $TARGET') \ No newline at end of file -- cgit v0.12 From d4f6b6a5903311f4e7c263272ae77020a0332a82 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Tue, 5 Oct 2021 18:39:32 -0700 Subject: [ci skip] Fix sider complain --- test/ninja/force_scons_callback.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/ninja/force_scons_callback.py b/test/ninja/force_scons_callback.py index 44cb7ec..55c12ca 100644 --- a/test/ninja/force_scons_callback.py +++ b/test/ninja/force_scons_callback.py @@ -23,7 +23,6 @@ # import os -import sys import TestSCons from TestCmd import IS_WINDOWS -- cgit v0.12 From 9fde9654191ccc21873710143357c3ca580fe72b Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Mon, 4 Oct 2021 13:03:47 -0600 Subject: Tweak Variables incl. manpage, docstrings * Added link anchors in variables-related funcs/methods, and link to them - these methods are not part of the generated link setup. * Clarified that vars set to defaults are not saved. * Updated docstrings in the Variables source (for API docs). * Added return-type annotations to Variables. * Fix for converter function possibly failing if it tries to access an environment. Fixes #2064. * Fixed up the behavior of aliases to variables, and added docu. Fixes #3869. * Fix PathIsDirCreate validator to catch permission problems. Fixes #2828 Signed-off-by: Mats Wichmann --- CHANGES.txt | 8 ++ SCons/Variables/BoolVariable.py | 67 ++++++++------ SCons/Variables/EnumVariable.py | 51 +++++------ SCons/Variables/ListVariable.py | 74 +++++++++------- SCons/Variables/PackageVariable.py | 81 ++++++++--------- SCons/Variables/PathVariable.py | 122 ++++++++++++++------------ SCons/Variables/PathVariableTests.py | 16 +++- SCons/Variables/VariablesTests.py | 96 +++++++++++---------- SCons/Variables/__init__.py | 163 ++++++++++++++++++----------------- doc/man/scons.xml | 127 ++++++++++++++++----------- 10 files changed, 453 insertions(+), 352 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 3e26ade..6a708e0 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -53,6 +53,14 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER Added new sconsign filenames to skip_entry_list in Scanner/Dir.py - Change SCons.Scanner.Base to ScannerBase. Old name kept as an alias but is now unused in SCons itself. + - Call Variables option converter consistently - the converter should + have access to the env if it needs to (issue #2064). + - Fixed the variables Add() method to accept a tuple for the variable + name the same way AddVariables() does (issue #3869). + - The premade validator PathIsDirCreate for for PathVariable now catches + the case where the directory could not be created due to permission + problems, allowing a more helpful error to be emitted (issue #2828) + RELEASE 4.2.0 - Sat, 31 Jul 2021 18:12:46 -0700 diff --git a/SCons/Variables/BoolVariable.py b/SCons/Variables/BoolVariable.py index 6d9ff72..b3c0dc4 100644 --- a/SCons/Variables/BoolVariable.py +++ b/SCons/Variables/BoolVariable.py @@ -21,7 +21,7 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -"""Option type for true/false Variables. +"""Variable type for true/false Variables. Usage example:: @@ -32,51 +32,62 @@ Usage example:: ... """ -__all__ = ['BoolVariable',] +from typing import Tuple, Callable import SCons.Errors -__true_strings = ('y', 'yes', 'true', 't', '1', 'on' , 'all' ) -__false_strings = ('n', 'no', 'false', 'f', '0', 'off', 'none') +__all__ = ['BoolVariable',] +TRUE_STRINGS = ('y', 'yes', 'true', 't', '1', 'on' , 'all' ) +FALSE_STRINGS = ('n', 'no', 'false', 'f', '0', 'off', 'none') -def _text2bool(val): - """ - Converts strings to True/False depending on the 'truth' expressed by - the string. If the string can't be converted, the original value - will be returned. - See '__true_strings' and '__false_strings' for values considered - 'true' or 'false respectively. +def _text2bool(val) -> bool: + """Converts strings to True/False. - This is usable as 'converter' for SCons' Variables. + If *val* looks like it expresses a bool-like value, based on + the :data:`TRUE_STRINGS` and :data:`FALSE_STRINGS` tuples, + return the appropriate value. + + This is usable as a converter function for SCons Variables. + + Raises: + ValueError: if the string cannot be converted. """ + lval = val.lower() - if lval in __true_strings: return True - if lval in __false_strings: return False + if lval in TRUE_STRINGS: + return True + if lval in FALSE_STRINGS: + return False raise ValueError("Invalid value for boolean option: %s" % val) -def _validator(key, val, env): - """ - Validates the given value to be either '0' or '1'. - - This is usable as 'validator' for SCons' Variables. +def _validator(key, val, env) -> None: + """Validates the given value to be either true or false. + + This is usable as a validator function for SCons Variables. + + Raises: + KeyError: if key is not set in env + UserError: if key does not validate. """ if not env[key] in (True, False): raise SCons.Errors.UserError( - 'Invalid value for boolean option %s: %s' % (key, env[key])) + 'Invalid value for boolean option %s: %s' % (key, env[key]) + ) -def BoolVariable(key, help, default): - """ - The input parameters describe a boolean option, thus they are - returned with the correct converter and validator appended. The - 'help' text will by appended by '(yes|no) to show the valid - valued. The result is usable for input to opts.Add(). +def BoolVariable(key, help, default) -> Tuple[str, str, str, Callable, Callable]: + """Return a tuple describing a boolean SCons Variable. + + The input parameters describe a boolean option. Returns a tuple + including the correct converter and validator. + The *help* text will have ``(yes|no)`` automatically appended to show the + valid values. The result is usable as input to :meth:`Add`. """ - return (key, '%s (yes|no)' % help, default, - _validator, _text2bool) + help = '%s (yes|no)' % help + return (key, help, default, _validator, _text2bool) # Local Variables: # tab-width:4 diff --git a/SCons/Variables/EnumVariable.py b/SCons/Variables/EnumVariable.py index 6e0a8c5..e39eb02 100644 --- a/SCons/Variables/EnumVariable.py +++ b/SCons/Variables/EnumVariable.py @@ -21,10 +21,9 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -"""Option type for enumeration Variables. +"""Variable type for enumeration Variables. -This file defines the option type for SCons allowing only specified -input-values. +Enumeration variables allow selection of one from a specified set of values. Usage example:: @@ -32,8 +31,8 @@ Usage example:: opts.Add( EnumVariable( 'debug', - 'debug output and symbols', - 'no', + help='debug output and symbols', + default='no', allowed_values=('yes', 'no', 'full'), map={}, ignorecase=2, @@ -44,48 +43,52 @@ Usage example:: ... """ +from typing import Tuple, Callable -__all__ = ['EnumVariable',] +import SCons.Errors +__all__ = ['EnumVariable',] -import SCons.Errors -def _validator(key, val, env, vals): +def _validator(key, val, env, vals) -> None: if val not in vals: raise SCons.Errors.UserError( 'Invalid value for option %s: %s. Valid values are: %s' % (key, val, vals)) -def EnumVariable(key, help, default, allowed_values, map={}, ignorecase=0): - """ +def EnumVariable(key, help, default, allowed_values, map={}, ignorecase=0) -> Tuple[str, str, str, Callable, Callable]: + """Return a tuple describing an enumaration SCons Variable. + The input parameters describe an option with only certain values - allowed. They are returned with an appropriate converter and - validator appended. The result is usable for input to - Variables.Add(). + allowed. Returns A tuple including an appropriate converter and + validator. The result is usable as input to :meth:`Add`. - 'key' and 'default' are the values to be passed on to Variables.Add(). + *key* and *default* are passed directly on to :meth:`Add`. - 'help' will be appended by the allowed values automatically + *help* is the descriptive part of the help text, + and will have the allowed values automatically appended. - 'allowed_values' is a list of strings, which are allowed as values + *allowed_values* is a list of strings, which are the allowed values for this option. - The 'map'-dictionary may be used for converting the input value + The *map*-dictionary may be used for converting the input value into canonical values (e.g. for aliases). - 'ignorecase' defines the behaviour of the validator: + The value of *ignorecase* defines the behaviour of the validator: - If ignorecase == 0, the validator/converter are case-sensitive. - If ignorecase == 1, the validator/converter are case-insensitive. - If ignorecase == 2, the validator/converter is case-insensitive and the converted value will always be lower-case. + * 0: the validator/converter are case-sensitive. + * 1: the validator/converter are case-insensitive. + * 2: the validator/converter is case-insensitive and the + converted value will always be lower-case. - The 'validator' tests whether the value is in the list of allowed values. The 'converter' converts input values - according to the given 'map'-dictionary (unmapped input values are returned unchanged). + The *validator* tests whether the value is in the list of allowed values. + The *converter* converts input values according to the given + *map*-dictionary (unmapped input values are returned unchanged). """ help = '%s (%s)' % (help, '|'.join(allowed_values)) # define validator - if ignorecase >= 1: + if ignorecase: validator = lambda key, val, env: \ _validator(key, val.lower(), env, allowed_values) else: diff --git a/SCons/Variables/ListVariable.py b/SCons/Variables/ListVariable.py index c615258..7bd6053 100644 --- a/SCons/Variables/ListVariable.py +++ b/SCons/Variables/ListVariable.py @@ -21,9 +21,7 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -"""Option type for list Variables. - -This file defines the option type for SCons implementing 'lists'. +"""Variable type for list Variables. A 'list' option may either be 'all', 'none' or a list of names separated by comma. After the option has been processed, the option @@ -38,8 +36,8 @@ Usage example:: opts.Add( ListVariable( 'shared', - 'libraries to build as shared libraries', - 'all', + help='libraries to build as shared libraries', + default='all', elems=list_of_libs, ) ) @@ -54,44 +52,55 @@ Usage example:: # Known Bug: This should behave like a Set-Type, but does not really, # since elements can occur twice. -__all__ = ['ListVariable',] - import collections +from typing import Tuple, Callable import SCons.Util +__all__ = ['ListVariable',] + class _ListVariable(collections.UserList): - def __init__(self, initlist=[], allowedElems=[]): - collections.UserList.__init__(self, [_f for _f in initlist if _f]) + def __init__(self, initlist=None, allowedElems=None): + if initlist is None: + initlist = [] + if allowedElems is None: + allowedElems = [] + super().__init__([_f for _f in initlist if _f]) self.allowedElems = sorted(allowedElems) def __cmp__(self, other): raise NotImplementedError + def __eq__(self, other): raise NotImplementedError + def __ge__(self, other): raise NotImplementedError + def __gt__(self, other): raise NotImplementedError + def __le__(self, other): raise NotImplementedError + def __lt__(self, other): raise NotImplementedError + def __str__(self): - if len(self) == 0: + if not len(self): return 'none' self.data.sort() if self.data == self.allowedElems: return 'all' else: return ','.join(self) + def prepare_to_store(self): return self.__str__() -def _converter(val, allowedElems, mapdict): - """ - """ +def _converter(val, allowedElems, mapdict) -> _ListVariable: + """ """ if val == 'none': val = [] elif val == 'all': @@ -101,35 +110,40 @@ def _converter(val, allowedElems, mapdict): val = [mapdict.get(v, v) for v in val] notAllowed = [v for v in val if v not in allowedElems] if notAllowed: - raise ValueError("Invalid value(s) for option: %s" % - ','.join(notAllowed)) + raise ValueError( + "Invalid value(s) for option: %s" % ','.join(notAllowed) + ) return _ListVariable(val, allowedElems) -## def _validator(key, val, env): -## """ -## """ -## # todo: write validator for pgk list -## return 1 +# def _validator(key, val, env) -> None: +# """ """ +# # TODO: write validator for pgk list +# pass -def ListVariable(key, help, default, names, map={}): - """ - The input parameters describe a 'package list' option, thus they - are returned with the correct converter and validator appended. The - result is usable for input to opts.Add() . +def ListVariable(key, help, default, names, map={}) -> Tuple[str, str, str, None, Callable]: + """Return a tuple describing a list SCons Variable. + + The input parameters describe a 'list' option. Returns + a tuple including the correct converter and validator. + The result is usable for input to :meth:`Add`. + + *help* will have text appended indicating the legal values + (not including any extra names from *map*). + + *map* can be used to map alternative names to the ones in *names* - + that is, a form of alias. - A 'package list' option may either be 'all', 'none' or a list of - package names (separated by space). + A 'list' option may either be 'all', 'none' or a list of + names (separated by commas). """ names_str = 'allowed names: %s' % ' '.join(names) if SCons.Util.is_List(default): default = ','.join(default) help = '\n '.join( (help, '(all|none|comma-separated list of names)', names_str)) - return (key, help, default, - None, #_validator, - lambda val: _converter(val, names, map)) + return (key, help, default, None, lambda val: _converter(val, names, map)) # Local Variables: # tab-width:4 diff --git a/SCons/Variables/PackageVariable.py b/SCons/Variables/PackageVariable.py index d4dafd5..43ce99d 100644 --- a/SCons/Variables/PackageVariable.py +++ b/SCons/Variables/PackageVariable.py @@ -21,58 +21,58 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -"""Option type for package Variables. - -This file defines the option type for SCons implementing 'package activation'. +"""Variable type for package Variables. To be used whenever a 'package' may be enabled/disabled and the package path may be specified. -Usage example: - - Examples: - x11=no (disables X11 support) - x11=yes (will search for the package installation dir) - x11=/usr/local/X11 (will check this path for existence) - - To replace autoconf's --with-xxx=yyy :: - - - opts = Variables() - opts.Add(PackageVariable('x11', - 'use X11 installed here (yes = search some places', - 'yes')) - ... - if env['x11'] == True: - dir = ... search X11 in some standard places ... - env['x11'] = dir - if env['x11']: - ... build with x11 ... +Given these options :: + + x11=no (disables X11 support) + x11=yes (will search for the package installation dir) + x11=/usr/local/X11 (will check this path for existence) + +Can be used as a replacement for autoconf's ``--with-xxx=yyy`` :: + + opts = Variables() + opts.Add( + PackageVariable( + key='x11', + help='use X11 installed here (yes = search some places)', + default='yes' + ) + ) + ... + if env['x11'] == True: + dir = ... # search X11 in some standard places ... + env['x11'] = dir + if env['x11']: + ... # build with x11 ... """ -__all__ = ['PackageVariable',] +from typing import Tuple, Callable import SCons.Errors +__all__ = ['PackageVariable',] + __enable_strings = ('1', 'yes', 'true', 'on', 'enable', 'search') __disable_strings = ('0', 'no', 'false', 'off', 'disable') -def _converter(val): - """ - """ +def _converter(val) -> bool: + """ """ lval = val.lower() if lval in __enable_strings: return True if lval in __disable_strings: return False - #raise ValueError("Invalid value for boolean option: %s" % val) return val -def _validator(key, val, env, searchfunc): +def _validator(key, val, env, searchfunc) -> None: + """ """ # NB: searchfunc is currently undocumented and unsupported - """ - """ # TODO write validator, check for path import os + if env[key] is True: if searchfunc: env[key] = searchfunc(key, val) @@ -81,20 +81,21 @@ def _validator(key, val, env, searchfunc): 'Path does not exist for option %s: %s' % (key, val)) -def PackageVariable(key, help, default, searchfunc=None): - # NB: searchfunc is currently undocumented and unsupported - """ - The input parameters describe a 'package list' option, thus they - are returned with the correct converter and validator appended. The - result is usable for input to opts.Add() . +def PackageVariable(key, help, default, searchfunc=None) -> Tuple[str, str, str, Callable, Callable]: + """Return a tuple describing a package list SCons Variable. - A 'package list' option may either be 'all', 'none' or a list of - package names (separated by space). + The input parameters describe a 'package list' option. Returns + a tuple including the correct converter and validator appended. + The result is usable as input to :meth:`Add` . + + A 'package list' option may either be 'all', 'none' or a pathname + string. This information is appended to *help*. """ + # NB: searchfunc is currently undocumented and unsupported help = '\n '.join( (help, '( yes | no | /path/to/%s )' % key)) return (key, help, default, - lambda k, v, e: _validator(k,v,e,searchfunc), + lambda k, v, e: _validator(k, v, e, searchfunc), _converter) # Local Variables: diff --git a/SCons/Variables/PathVariable.py b/SCons/Variables/PathVariable.py index 081872b..383c1f9 100644 --- a/SCons/Variables/PathVariable.py +++ b/SCons/Variables/PathVariable.py @@ -21,64 +21,74 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -"""Option type for path Variables. +"""Variable type for path Variables. -This file defines an option type for SCons implementing path settings. - -To be used whenever a user-specified path override should be allowed. +To be used whenever a user-specified path override setting should be allowed. Arguments to PathVariable are: - option-name = name of this option on the command line (e.g. "prefix") - option-help = help string for option - option-dflt = default value for this option - validator = [optional] validator for option value. Predefined are: - PathAccept -- accepts any path setting; no validation - PathIsDir -- path must be an existing directory - PathIsDirCreate -- path must be a dir; will create - PathIsFile -- path must be a file - PathExists -- path must exist (any type) [default] - - The validator is a function that is called and which should return + * *key* - name of this option on the command line (e.g. "prefix") + * *help* - help string for option + * *dflt* - default value for this option + * *validator* - [optional] validator for option value. Predefined are: + + * *PathAccept* - accepts any path setting; no validation + * *PathIsDir* - path must be an existing directory + * *PathIsDirCreate* - path must be a dir; will create + * *PathIsFile* - path must be a file + * *PathExists* - path must exist (any type) [default] + + The *validator* is a function that is called and which should return True or False to indicate if the path is valid. The arguments - to the validator function are: (key, val, env). The key is the - name of the option, the val is the path specified for the option, - and the env is the env to which the Options have been added. + to the validator function are: (*key*, *val*, *env*). *key* is the + name of the option, *val* is the path specified for the option, + and *env* is the environment to which the Options have been added. Usage example:: - Examples: - prefix=/usr/local - - opts = Variables() - - opts = Variables() - opts.Add(PathVariable('qtdir', - 'where the root of Qt is installed', - qtdir, PathIsDir)) - opts.Add(PathVariable('qt_includes', - 'where the Qt includes are installed', - '$qtdir/includes', PathIsDirCreate)) - opts.Add(PathVariable('qt_libraries', - 'where the Qt library is installed', - '$qtdir/lib')) + opts = Variables() + opts.Add( + PathVariable( + 'qtdir', + help='where the root of Qt is installed', + default=qtdir, + validator=PathIsDir, + ) + ) + opts.Add( + PathVariable( + 'qt_includes', + help='where the Qt includes are installed', + default='$qtdir/includes', + validator=PathIsDirCreate, + ) + ) + opts.Add( + PathVariable( + 'qt_libraries', + help='where the Qt library is installed', + default='$qtdir/lib', + ) + ) """ -__all__ = ['PathVariable',] import os import os.path +from typing import Tuple, Callable import SCons.Errors +__all__ = ['PathVariable',] + class _PathVariableClass: @staticmethod - def PathAccept(key, val, env): + def PathAccept(key, val, env) -> None: """Accepts any path, no checking done.""" pass - + @staticmethod - def PathIsDir(key, val, env): + def PathIsDir(key, val, env) -> None: """Validator to check if Path is a directory.""" if not os.path.isdir(val): if os.path.isfile(val): @@ -88,17 +98,20 @@ class _PathVariableClass: raise SCons.Errors.UserError(m % (key, val)) @staticmethod - def PathIsDirCreate(key, val, env): + def PathIsDirCreate(key, val, env) -> None: """Validator to check if Path is a directory, creating it if it does not exist.""" - if os.path.isfile(val): + try: + os.makedirs(val, exist_ok=True) + except FileExistsError: m = 'Path for option %s is a file, not a directory: %s' raise SCons.Errors.UserError(m % (key, val)) - if not os.path.isdir(val): - os.makedirs(val) + except PermissionError: + m = 'Path for option %s could not be created: %s' + raise SCons.Errors.UserError(m % (key, val)) @staticmethod - def PathIsFile(key, val, env): + def PathIsFile(key, val, env) -> None: """Validator to check if Path is a file""" if not os.path.isfile(val): if os.path.isdir(val): @@ -108,32 +121,33 @@ class _PathVariableClass: raise SCons.Errors.UserError(m % (key, val)) @staticmethod - def PathExists(key, val, env): + def PathExists(key, val, env) -> None: """Validator to check if Path exists""" if not os.path.exists(val): m = 'Path for option %s does not exist: %s' raise SCons.Errors.UserError(m % (key, val)) - def __call__(self, key, help, default, validator=None): - """ - The input parameters describe a 'path list' option, thus they - are returned with the correct converter and validator appended. The - result is usable for input to opts.Add() . + def __call__(self, key, help, default, validator=None) -> Tuple[str, str, str, Callable, None]: + """Return a tuple describing a path list SCons Variable. - The 'default' option specifies the default path to use if the + The input parameters describe a 'path list' option. Returns + a tuple with the correct converter and validator appended. The + result is usable for input to :meth:`Add`. + + The *default* option specifies the default path to use if the user does not specify an override with this option. - validator is a validator, see this file for examples + *validator* is a validator, see this file for examples """ if validator is None: validator = self.PathExists if SCons.Util.is_List(key) or SCons.Util.is_Tuple(key): - return (key, '%s ( /path/to/%s )' % (help, key[0]), default, - validator, None) + helpmsg = '%s ( /path/to/%s )' % (help, key[0]) else: - return (key, '%s ( /path/to/%s )' % (help, key), default, - validator, None) + helpmsg = '%s ( /path/to/%s )' % (help, key) + return (key, helpmsg, default, validator, None) + PathVariable = _PathVariableClass() diff --git a/SCons/Variables/PathVariableTests.py b/SCons/Variables/PathVariableTests.py index 6e3a70b..a9aa8f0 100644 --- a/SCons/Variables/PathVariableTests.py +++ b/SCons/Variables/PathVariableTests.py @@ -96,8 +96,8 @@ class PathVariableTestCase(unittest.TestCase): o.validator('X', dne, {}) except SCons.Errors.UserError as e: assert str(e) == 'Directory path for option X does not exist: %s' % dne, e - except: - raise Exception("did not catch expected UserError") + except Exception as e: + raise Exception("did not catch expected UserError") from e def test_PathIsDirCreate(self): """Test the PathIsDirCreate validator""" @@ -121,8 +121,16 @@ class PathVariableTestCase(unittest.TestCase): o.validator('X', f, {}) except SCons.Errors.UserError as e: assert str(e) == 'Path for option X is a file, not a directory: %s' % f, e - except: - raise Exception("did not catch expected UserError") + except Exception as e: + raise Exception("did not catch expected UserError") from e + + f = '/yyy/zzz' # this not exists and should fail to create + try: + o.validator('X', f, {}) + except SCons.Errors.UserError as e: + assert str(e) == 'Path for option X could not be created: %s' % f, e + except Exception as e: + raise Exception("did not catch expected UserError") from e def test_PathIsFile(self): """Test the PathIsFile validator""" diff --git a/SCons/Variables/VariablesTests.py b/SCons/Variables/VariablesTests.py index 8def19e..4672cdd 100644 --- a/SCons/Variables/VariablesTests.py +++ b/SCons/Variables/VariablesTests.py @@ -46,7 +46,7 @@ class Environment: def check(key, value, env): assert int(value) == 6 * 9, "key %s = %s" % (key, repr(value)) - + # Check saved option file by executing and comparing against # the expected dictionary def checkSave(file, expected): @@ -138,7 +138,7 @@ class VariablesTestCase(unittest.TestCase): test = TestSCons.TestSCons() file = test.workpath('custom.py') opts = SCons.Variables.Variables(file) - + opts.Add('ANSWER', 'THE answer to THE question', "42", @@ -159,7 +159,7 @@ class VariablesTestCase(unittest.TestCase): file = test.workpath('custom.py') test.write('custom.py', 'ANSWER=54') opts = SCons.Variables.Variables(file) - + opts.Add('ANSWER', 'THE answer to THE question', "42", @@ -187,7 +187,7 @@ class VariablesTestCase(unittest.TestCase): file = test.workpath('custom.py') test.write('custom.py', 'ANSWER=42') opts = SCons.Variables.Variables(file) - + opts.Add('ANSWER', 'THE answer to THE question', "10", @@ -208,7 +208,7 @@ class VariablesTestCase(unittest.TestCase): file = test.workpath('custom.py') test.write('custom.py', 'ANSWER=10') opts = SCons.Variables.Variables(file) - + opts.Add('ANSWER', 'THE answer to THE question', "12", @@ -229,7 +229,7 @@ class VariablesTestCase(unittest.TestCase): file = test.workpath('custom.py') test.write('custom.py', 'ANSWER=10') opts = SCons.Variables.Variables(file) - + opts.Add('ANSWER', 'THE answer to THE question', "12", @@ -247,7 +247,7 @@ class VariablesTestCase(unittest.TestCase): test = TestSCons.TestSCons() file = test.workpath('custom.py') opts = SCons.Variables.Variables(file) - + opts.Add('ANSWER', help='THE answer to THE question', converter=str) @@ -282,9 +282,9 @@ class VariablesTestCase(unittest.TestCase): nopts = SCons.Variables.Variables() # Ensure that both attributes are initialized to - # an empty list and dict, respectively. + # an empty list and dict, respectively. assert len(nopts.files) == 0 - assert len(nopts.args) == 0 + assert len(nopts.args) == 0 def test_args(self): """Test updating an Environment with arguments overridden""" @@ -295,7 +295,7 @@ class VariablesTestCase(unittest.TestCase): file = test.workpath('custom.py') test.write('custom.py', 'ANSWER=42') opts = SCons.Variables.Variables(file, {'ANSWER':54}) - + opts.Add('ANSWER', 'THE answer to THE question', "42", @@ -315,7 +315,7 @@ class VariablesTestCase(unittest.TestCase): file = test.workpath('custom.py') test.write('custom.py', 'ANSWER=54') opts = SCons.Variables.Variables(file, {'ANSWER':42}) - + opts.Add('ANSWER', 'THE answer to THE question', "54", @@ -332,7 +332,7 @@ class VariablesTestCase(unittest.TestCase): file = test.workpath('custom.py') test.write('custom.py', 'ANSWER=54') opts = SCons.Variables.Variables(file, {'ANSWER':54}) - + opts.Add('ANSWER', 'THE answer to THE question', "54", @@ -354,7 +354,7 @@ class VariablesTestCase(unittest.TestCase): if val in [1, 'y']: val = 1 if val in [0, 'n']: val = 0 return val - + # test saving out empty file opts.Add('OPT_VAL', 'An option to test', @@ -400,11 +400,11 @@ class VariablesTestCase(unittest.TestCase): self.x = x def __str__(self): return self.x - + test = TestSCons.TestSCons() cache_file = test.workpath('cached.options') opts = SCons.Variables.Variables() - + opts.Add('THIS_USED_TO_BREAK', 'An option to test', "Default") @@ -412,11 +412,11 @@ class VariablesTestCase(unittest.TestCase): opts.Add('THIS_ALSO_BROKE', 'An option to test', "Default2") - + opts.Add('THIS_SHOULD_WORK', 'An option to test', Foo('bar')) - + env = Environment() opts.Update(env, { 'THIS_USED_TO_BREAK' : "Single'Quotes'In'String", 'THIS_ALSO_BROKE' : "\\Escape\nSequences\t", @@ -502,7 +502,10 @@ A: a - alpha test assert text == expectAlpha, text textBackwards = opts.GenerateHelpText(env, sort=lambda x, y: cmp(y, x)) - assert textBackwards == expectBackwards, "Expected:\n%s\nGot:\n%s\n"%(textBackwards, expectBackwards) + assert textBackwards == expectBackwards, "Expected:\n%s\nGot:\n%s\n" % ( + textBackwards, + expectBackwards, + ) def test_FormatVariableHelpText(self): """Test generating custom format help text""" @@ -550,7 +553,7 @@ B 42 54 b - alpha test ['B'] """ text = opts.GenerateHelpText(env, sort=cmp) assert text == expectAlpha, text - + def test_Aliases(self): """Test option aliases""" # test alias as a tuple @@ -560,17 +563,17 @@ B 42 54 b - alpha test ['B'] 'THE answer to THE question', "42"), ) - + env = Environment() opts.Update(env, {'ANSWER' : 'answer'}) - + assert 'ANSWER' in env - + env = Environment() opts.Update(env, {'ANSWERALIAS' : 'answer'}) - + assert 'ANSWER' in env and 'ANSWERALIAS' not in env - + # test alias as a list opts = SCons.Variables.Variables() opts.AddVariables( @@ -578,17 +581,16 @@ B 42 54 b - alpha test ['B'] 'THE answer to THE question', "42"), ) - + env = Environment() opts.Update(env, {'ANSWER' : 'answer'}) - + assert 'ANSWER' in env - + env = Environment() opts.Update(env, {'ANSWERALIAS' : 'answer'}) - - assert 'ANSWER' in env and 'ANSWERALIAS' not in env + assert 'ANSWER' in env and 'ANSWERALIAS' not in env class UnknownVariablesTestCase(unittest.TestCase): @@ -596,7 +598,7 @@ class UnknownVariablesTestCase(unittest.TestCase): def test_unknown(self): """Test the UnknownVariables() method""" opts = SCons.Variables.Variables() - + opts.Add('ANSWER', 'THE answer to THE question', "42") @@ -612,38 +614,38 @@ class UnknownVariablesTestCase(unittest.TestCase): r = opts.UnknownVariables() assert r == {'UNKNOWN' : 'unknown'}, r assert env['ANSWER'] == 'answer', env['ANSWER'] - + def test_AddOptionUpdatesUnknown(self): """Test updating of the 'unknown' dict""" opts = SCons.Variables.Variables() - + opts.Add('A', 'A test variable', "1") - + args = { 'A' : 'a', 'ADDEDLATER' : 'notaddedyet', } - + env = Environment() opts.Update(env,args) - + r = opts.UnknownVariables() assert r == {'ADDEDLATER' : 'notaddedyet'}, r assert env['A'] == 'a', env['A'] - + opts.Add('ADDEDLATER', 'An option not present initially', "1") - + args = { 'A' : 'a', 'ADDEDLATER' : 'added', } - + opts.Update(env, args) - + r = opts.UnknownVariables() assert len(r) == 0, r assert env['ADDEDLATER'] == 'added', env['ADDEDLATER'] @@ -651,33 +653,33 @@ class UnknownVariablesTestCase(unittest.TestCase): def test_AddOptionWithAliasUpdatesUnknown(self): """Test updating of the 'unknown' dict (with aliases)""" opts = SCons.Variables.Variables() - + opts.Add('A', 'A test variable', "1") - + args = { 'A' : 'a', 'ADDEDLATERALIAS' : 'notaddedyet', } - + env = Environment() opts.Update(env,args) - + r = opts.UnknownVariables() assert r == {'ADDEDLATERALIAS' : 'notaddedyet'}, r assert env['A'] == 'a', env['A'] - + opts.AddVariables( (('ADDEDLATER', 'ADDEDLATERALIAS'), 'An option not present initially', "1"), ) - + args['ADDEDLATERALIAS'] = 'added' - + opts.Update(env, args) - + r = opts.UnknownVariables() assert len(r) == 0, r assert env['ADDEDLATER'] == 'added', env['ADDEDLATER'] diff --git a/SCons/Variables/__init__.py b/SCons/Variables/__init__.py index 4a30a91..9c8bdca 100644 --- a/SCons/Variables/__init__.py +++ b/SCons/Variables/__init__.py @@ -44,7 +44,7 @@ class Variables: Holds all the options, updates the environment with the variables, and renders the help text. - If is_global is True, this is a singleton, create only once. + If *is_global* is true, this is a singleton, create only once. Args: files (optional): List of option configuration files to load @@ -64,7 +64,7 @@ class Variables: self.args = args if not SCons.Util.is_List(files): if files: - files = [ files ] + files = [files,] else: files = [] self.files = files @@ -77,20 +77,23 @@ class Variables: if not Variables.instance: Variables.instance=self - def _do_add(self, key, help="", default=None, validator=None, converter=None): + def _do_add(self, key, help="", default=None, validator=None, converter=None, **kwargs) -> None: class Variable: pass option = Variable() - # if we get a list or a tuple, we take the first element as the + # If we get a list or a tuple, we take the first element as the # option key and store the remaining in aliases. if SCons.Util.is_List(key) or SCons.Util.is_Tuple(key): - option.key = key[0] - option.aliases = key[1:] + option.key = key[0] + option.aliases = list(key[1:]) else: - option.key = key - option.aliases = [ key ] + option.key = key + # TODO: normalize to not include key in aliases. Currently breaks tests. + option.aliases = [key,] + if not SCons.Environment.is_valid_construction_var(option.key): + raise SCons.Errors.UserError("Illegal Variables key `%s'" % str(option.key)) option.help = help option.default = default option.validator = validator @@ -100,43 +103,42 @@ class Variables: # options might be added after the 'unknown' dict has been set up, # so we remove the key and all its aliases from that dict - for alias in list(option.aliases) + [ option.key ]: + for alias in option.aliases + [option.key,]: if alias in self.unknown: del self.unknown[alias] - def keys(self): - """ - Returns the keywords for the options - """ + def keys(self) -> list: + """Returns the keywords for the options.""" return [o.key for o in self.options] - def Add(self, key, help="", default=None, validator=None, converter=None, **kw): - r"""Add an option. + def Add(self, key, *args, **kwargs) -> None: + r""" Add an option. Args: - key: the name of the variable, or a list or tuple of arguments - help: optional help text for the options (Default value = "") - default: optional default value for option (Default value = None) - validator: optional function called to validate the option's value - (Default value = None) - converter: optional function to be called to convert the option's - value before putting it in the environment. (Default value = None) - \*\*kw: keyword args, unused. + key: the name of the variable, or a 5-tuple (or list). + If a tuple, and there are no additional arguments, + the tuple is unpacked into help, default, validator, converter. + If there are additional arguments, the first word of the tuple + is taken as the key, and the remainder as aliases. + \*args: optional positional arguments + help: optional help text for the options (Default value = "") + default: optional default value for option (Default value = None) + validator: optional function called to validate the option's value + (Default value = None) + converter: optional function to be called to convert the option's + value before putting it in the environment. (Default value = None) + \*\*kwargs: keyword args, can be the arguments from \*args or + arbitrary kwargs used by a variable itself """ - if SCons.Util.is_List(key) or isinstance(key, tuple): - self._do_add(*key) - return - - if not SCons.Util.is_String(key) or \ - not SCons.Environment.is_valid_construction_var(key): - raise SCons.Errors.UserError("Illegal Variables.Add() key `%s'" % str(key)) + if SCons.Util.is_List(key) or SCons.Util.is_Tuple(key): + if not (len(args) or len(kwargs)): + return self._do_add(*key) - self._do_add(key, help, default, validator, converter) + return self._do_add(key, *args, **kwargs) - def AddVariables(self, *optlist): - """ - Add a list of options. + def AddVariables(self, *optlist) -> None: + """ Add a list of options. Each list element is a tuple/list of arguments to be passed on to the underlying method for adding options. @@ -154,12 +156,13 @@ class Variables: for o in optlist: self._do_add(*o) + def Update(self, env, args=None) -> None: + """ Update an environment with the option variables. - def Update(self, env, args=None): - """ - Update an environment with the option variables. - - env - the environment to update. + Args: + env: the environment to update. + args: [optional] a dictionary of keys and values to update + in *env*. If omitted, uses the variables from the commandline. """ values = {} @@ -192,7 +195,7 @@ class Variables: for arg, value in args.items(): added = False for option in self.options: - if arg in list(option.aliases) + [ option.key ]: + if arg in option.aliases + [option.key,]: values[option.key] = value added = True if not added: @@ -206,7 +209,7 @@ class Variables: except KeyError: pass - # Call the convert functions: + # apply converters for option in self.options: if option.converter and option.key in values: value = env.subst('${%s}'%option.key) @@ -219,36 +222,39 @@ class Variables: raise SCons.Errors.UserError('Error converting option: %s\n%s'%(option.key, x)) - # Finally validate the values: + # apply validators for option in self.options: if option.validator and option.key in values: option.validator(option.key, env.subst('${%s}'%option.key), env) - def UnknownVariables(self): - """ - Returns any options in the specified arguments lists that - were not known, declared options in this object. + def UnknownVariables(self) -> dict: + """ Returns unknown variables. + + Identifies options that were not known, declared options in this object. """ return self.unknown - def Save(self, filename, env): - """ - Saves all the options in the given file. This file can - then be used to load the options next run. This can be used - to create an option cache file. + def Save(self, filename, env) -> None: + """ Save the options to a file. + + Saves all the options which have non-default settings + to the given file as Python expressions. This file can + then be used to load the options for a subsequent run. + This can be used to create an option cache file. - filename - Name of the file to save into - env - the environment get the option values from + Args: + filename: Name of the file to save into + env: the environment get the option values from """ # Create the file and write out the header try: - fh = open(filename, 'w') - - try: + with open(filename, 'w') as fh: # Make an assignment in the file for each option # within the environment that was assigned a value - # other than the default. + # other than the default. We don't want to save the + # ones set to default: in case the SConscript settings + # change you would then pick up old defaults. for option in self.options: try: value = env[option.key] @@ -268,55 +274,58 @@ class Variables: defaultVal = env.subst(SCons.Util.to_String(option.default)) if option.converter: - defaultVal = option.converter(defaultVal) + try: + defaultVal = option.converter(defaultVal) + except TypeError: + defaultVal = option.converter(defaultVal, env) if str(env.subst('${%s}' % option.key)) != str(defaultVal): fh.write('%s = %s\n' % (option.key, repr(value))) except KeyError: pass - finally: - fh.close() - except IOError as x: raise SCons.Errors.UserError('Error writing options to file: %s\n%s' % (filename, x)) - def GenerateHelpText(self, env, sort=None): - """ - Generate the help text for the options. + def GenerateHelpText(self, env, sort=None) -> str: + """ Generate the help text for the options. - env - an environment that is used to get the current values - of the options. - cmp - Either a function as follows: The specific sort function should take two arguments and return -1, 0 or 1 - or a boolean to indicate if it should be sorted. + Args: + env: an environment that is used to get the current values + of the options. + cmp: Either a comparison function used for sorting + (must take two arguments and return -1, 0 or 1) + or a boolean to indicate if it should be sorted. """ if callable(sort): - options = sorted(self.options, key=cmp_to_key(lambda x,y: sort(x.key,y.key))) + options = sorted(self.options, key=cmp_to_key(lambda x, y: sort(x.key, y.key))) elif sort is True: options = sorted(self.options, key=lambda x: x.key) else: options = self.options - def format(opt, self=self, env=env): + def format_opt(opt, self=self, env=env) -> str: if opt.key in env: actual = env.subst('${%s}' % opt.key) else: actual = None return self.FormatVariableHelpText(env, opt.key, opt.help, opt.default, actual, opt.aliases) - lines = [_f for _f in map(format, options) if _f] + lines = [_f for _f in map(format_opt, options) if _f] return ''.join(lines) - format = '\n%s: %s\n default: %s\n actual: %s\n' - format_ = '\n%s: %s\n default: %s\n actual: %s\n aliases: %s\n' + fmt = '\n%s: %s\n default: %s\n actual: %s\n' + aliasfmt = '\n%s: %s\n default: %s\n actual: %s\n aliases: %s\n' - def FormatVariableHelpText(self, env, key, help, default, actual, aliases=[]): + def FormatVariableHelpText(self, env, key, help, default, actual, aliases=None) -> str: + if aliases is None: + aliases = [] # Don't display the key name itself as an alias. aliases = [a for a in aliases if a != key] - if len(aliases)==0: - return self.format % (key, help, default, actual) + if aliases: + return self.aliasfmt % (key, help, default, actual, aliases) else: - return self.format_ % (key, help, default, actual, aliases) + return self.fmt % (key, help, default, actual) # Local Variables: # tab-width:4 diff --git a/doc/man/scons.xml b/doc/man/scons.xml index 214f38a..d38c6b7 100644 --- a/doc/man/scons.xml +++ b/doc/man/scons.xml @@ -4429,11 +4429,11 @@ from various sources, often from the command line: call the &Variables; function: - + Variables([files, [args]]) If files is a file or -list of files, those are executed as Python scripts, +list of files, they are executed as Python scripts, and the values of (global) Python variables set in those files are added as &consvars; in the &DefEnv;. If no files are specified, @@ -4460,10 +4460,14 @@ CC = 'my_cc' is specified, it is a dictionary of values that will override anything read from files. -This is primarily intended to pass the +The primary use is to pass the &ARGUMENTS; dictionary that holds variables -specified on the command line. -Example: +specified on the command line, +allowing you to indicate that if a setting appears +on both the command line and in the file(s), +the command line setting takes precedence. +However, any dictionary can be passed. +Examples: vars = Variables('custom.py') @@ -4492,36 +4496,59 @@ pre-defined &consvars; that your project will make use of Variables objects have the following methods: - + vars.Add(key, [help, default, validator, converter]) Add a customizable &consvar; to the Variables object. key -is the name of the variable. +is either the name of the variable, +or a tuple (or list), in which case +the first item in the tuple is taken as the variable name, +and any remaining values are considered aliases for the variable. help -is the help text for the variable. +is the help text for the variable +(default empty string). default -is the default value of the variable; -if the default value is +is the default value of the variable +(default None). +If default is None -and there is no explicit value specified, +and a value is not specified, the &consvar; will not be added to the &consenv;. -If set, validator -is called to validate the value of the variable. -A function supplied as a validator shall accept -arguments: key, -value, and env. -The recommended way to handle an invalid value is -to raise an exception (see example below). -If set, converter -is called to convert the value before putting it in the environment, and -should take either a value, or the value and environment, as parameters. + + +As a special case, if key +is a tuple (or list) and is the only +argument, the tuple is unpacked into the five parameters +listed above left to right, with any missing members filled with +the respecitive default values. This form allows Add +to consume a tuple emitted by the convenience functions +BoolVariable, +EnumVariable, +ListVariable, +PackageVariable +and +PathVariable. + + +If the optional validator is supplied, +it is called to validate the value of the variable. +A function supplied as a validator must accept +three arguments: key, +value and env, +and should raise an exception with a helpful error message +if value is invalid. +No return value is expected from the validator. + + +If the optional converter is supplied, +it is called to convert the value before putting it in the environment, +and should take either a value +or a value and environment as parameters. The converter function must return a value, -which will be converted into a string -before being validated by the -validator -(if any) +which will be converted into a string and be passed to the +validator (if any) and then added to the &consenv;. Examples: @@ -4538,11 +4565,11 @@ vars.Add('COLOR', validator=valid_color) - + vars.AddVariables(args) A convenience method that adds -multiple customizable &consvars; +one or more customizable &consvars; to a Variables object in one call; equivalent to calling &Add; multiple times. The args @@ -4551,10 +4578,10 @@ that contain the arguments for an individual call to the &Add; method. Since tuples are not Python mappings, the arguments cannot use the keyword form, -but rather are positional arguments as documented -for &Add;: a required name, the rest optional -but must be in the specified in order if used. - +but rather are positional arguments as documented for +Add: +a required name, the other four optional, +but must be in the specified order if used. @@ -4568,18 +4595,18 @@ opt.AddVariables( - + vars.Update(env, [args]) Update a &consenv; env -with the customized &consvars; . +with the customized &consvars;. Any specified variables that are not configured for the Variables object will be saved and may be retrieved using the -&UnknownVariables; -method, below. +&UnknownVariables; +method. Normally this method is not called directly, but rather invoked indirectly by passing the Variables object to @@ -4592,7 +4619,7 @@ env = Environment(variables=vars) - + vars.UnknownVariables() Returns a dictionary containing any @@ -4611,14 +4638,17 @@ for key, value in vars.UnknownVariables(): - + vars.Save(filename, env) Save the currently set variables into a script file named -by filename -that can be used on the next invocation to automatically load the current -settings. This method combined with the Variables method can be used to -support caching of variables between runs. +by filename. Only variables that are +set to non-default values are saved. +You can load these saved settings on a subsequent run +by passing filename to the +&Variables; function, +providing a way to cache particular settings for reuse. + env = Environment() @@ -4631,7 +4661,7 @@ vars.Save('variables.cache', env) - + vars.GenerateHelpText(env, [sort]) Generate help text documenting the customizable construction @@ -4663,7 +4693,7 @@ Help(vars.GenerateHelpText(env, sort=cmp)) - + vars.FormatVariableHelpText(env, opt, help, default, actual) Returns a formatted string @@ -4685,6 +4715,7 @@ string if you want the entries separated. def my_format(env, opt, help, default, actual): fmt = "\n%s: default=%s actual=%s (%s)\n" return fmt % (opt, default, actual, help) + vars.FormatVariableHelpText = my_format @@ -4701,7 +4732,7 @@ Each of these return a tuple ready to be passed to the &Add; or &AddVariables; method: - + BoolVariable(key, help, default) Returns a tuple of arguments @@ -4736,7 +4767,7 @@ as false. - + EnumVariable(key, help, default, allowed_values, [map, ignorecase]) Returns a tuple of arguments @@ -4785,7 +4816,7 @@ converted to lower case. - + ListVariable(key, help, default, names, [map]) Returns a tuple of arguments @@ -4824,7 +4855,7 @@ reflected in the generated help message). - + PackageVariable(key, help, default) Returns a tuple of arguments @@ -4864,7 +4895,7 @@ to disable use of the specified option. - + PathVariable(key, help, default, [validator]) Returns a tuple of arguments -- cgit v0.12 From c70ce59e8402272a077601b951b7a464d74cc807 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Fri, 8 Oct 2021 07:32:17 -0600 Subject: Update Job module to use thread.daemon Replaces call to deprecated thread.setDaemon. Signed-off-by: Mats Wichmann --- CHANGES.txt | 2 ++ SCons/Job.py | 9 +++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 3e26ade..23be598 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -53,6 +53,8 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER Added new sconsign filenames to skip_entry_list in Scanner/Dir.py - Change SCons.Scanner.Base to ScannerBase. Old name kept as an alias but is now unused in SCons itself. + - Maintenance: Python thread.setDaemon is deprecated in favor of + directly updating daemon attribute - update SCons to do this. RELEASE 4.2.0 - Sat, 31 Jul 2021 18:12:46 -0700 diff --git a/SCons/Job.py b/SCons/Job.py index f87a3bb..381b27a 100644 --- a/SCons/Job.py +++ b/SCons/Job.py @@ -33,6 +33,7 @@ import os import signal import SCons.Errors +import SCons.Warnings # The default stack size (in kilobytes) of the threads used to execute # jobs in parallel. @@ -139,7 +140,7 @@ class Jobs: self.job.taskmaster.stop() self.job.interrupted.set() else: - os._exit(2) + os._exit(2) # pylint: disable=protected-access self.old_sigint = signal.signal(signal.SIGINT, handler) self.old_sigterm = signal.signal(signal.SIGTERM, handler) @@ -231,7 +232,7 @@ else: def __init__(self, requestQueue, resultsQueue, interrupted): threading.Thread.__init__(self) - self.setDaemon(1) + self.daemon = True self.requestQueue = requestQueue self.resultsQueue = resultsQueue self.interrupted = interrupted @@ -389,7 +390,7 @@ else: if task.needs_execute(): # dispatch task self.tp.put(task) - jobs = jobs + 1 + jobs += 1 else: task.executed() task.postprocess() @@ -400,7 +401,7 @@ else: # back and put the next batch of tasks on the queue. while True: task, ok = self.tp.get() - jobs = jobs - 1 + jobs -= 1 if ok: task.executed() -- cgit v0.12 From 9d16d44e020755c907431d608d02551c5f4bca22 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Fri, 8 Oct 2021 14:09:47 -0600 Subject: Address review comments on Variables PR (#4031) Signed-off-by: Mats Wichmann --- SCons/Variables/BoolVariable.py | 2 +- SCons/Variables/PackageVariable.py | 12 +++++++----- SCons/Variables/PathVariable.py | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/SCons/Variables/BoolVariable.py b/SCons/Variables/BoolVariable.py index b3c0dc4..d4a7de7 100644 --- a/SCons/Variables/BoolVariable.py +++ b/SCons/Variables/BoolVariable.py @@ -38,7 +38,7 @@ import SCons.Errors __all__ = ['BoolVariable',] -TRUE_STRINGS = ('y', 'yes', 'true', 't', '1', 'on' , 'all' ) +TRUE_STRINGS = ('y', 'yes', 'true', 't', '1', 'on' , 'all') FALSE_STRINGS = ('n', 'no', 'false', 'f', '0', 'off', 'none') diff --git a/SCons/Variables/PackageVariable.py b/SCons/Variables/PackageVariable.py index 43ce99d..58c8441 100644 --- a/SCons/Variables/PackageVariable.py +++ b/SCons/Variables/PackageVariable.py @@ -56,14 +56,16 @@ import SCons.Errors __all__ = ['PackageVariable',] -__enable_strings = ('1', 'yes', 'true', 'on', 'enable', 'search') -__disable_strings = ('0', 'no', 'false', 'off', 'disable') +ENABLE_STRINGS = ('1', 'yes', 'true', 'on', 'enable', 'search') +DISABLE_STRINGS = ('0', 'no', 'false', 'off', 'disable') -def _converter(val) -> bool: +def _converter(val): """ """ lval = val.lower() - if lval in __enable_strings: return True - if lval in __disable_strings: return False + if lval in ENABLE_STRINGS: + return True + if lval in DISABLE_STRINGS: + return False return val diff --git a/SCons/Variables/PathVariable.py b/SCons/Variables/PathVariable.py index 383c1f9..a5bf6c5 100644 --- a/SCons/Variables/PathVariable.py +++ b/SCons/Variables/PathVariable.py @@ -28,7 +28,7 @@ To be used whenever a user-specified path override setting should be allowed. Arguments to PathVariable are: * *key* - name of this option on the command line (e.g. "prefix") * *help* - help string for option - * *dflt* - default value for this option + * *default* - default value for this option * *validator* - [optional] validator for option value. Predefined are: * *PathAccept* - accepts any path setting; no validation -- cgit v0.12