From fae4e0648cbc7632e1789ea44ee14f8be2654f39 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Fri, 29 Mar 2024 12:39:56 -0600 Subject: Tools update, maninly documentation Two small code changes: (1) if Environment() is called with a tools kwarg, reset TOOLS. Since it wasn't used in the init, any previous contents would be invalid for this env. (2) When adding the name of a tool to TOOLS, use AppendUnique rather than Append, so we get a uniqued list. This impacted one unit test, which uses a mock Envirionment with no AppendUnique method, so add one - the result is not used so the fact it's not doing any uniquing is ok. The rest is doc rewording and reorganization, and a little extra checking in the site_dir test. Signed-off-by: Mats Wichmann --- CHANGES.txt | 2 + RELEASE.txt | 1 + SCons/Environment.py | 23 +++- SCons/Environment.xml | 67 ++++++---- SCons/Tool/ToolTests.py | 1 + SCons/Tool/__init__.py | 2 +- doc/man/scons.xml | 305 ++++++++++++++++++++++++++------------------ test/site_scons/site-dir.py | 16 ++- 8 files changed, 260 insertions(+), 157 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index fc08c5e..c57a37a 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -30,6 +30,8 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Drop duplicated __getstate__ and __setstate__ methods in AliasNodeInfo, FileNodeInfo and ValueNodeInfo classes, as they are identical to the ones in parent NodeInfoBase and can just be inherited. + - Update manpage for Tools, and for TOOL, which also gets a minor + tweak for how it's handled (should be more accurate in a few situations). RELEASE 4.7.0 - Sun, 17 Mar 2024 17:22:20 -0700 diff --git a/RELEASE.txt b/RELEASE.txt index 5f8e9d5..5473bc0 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -62,6 +62,7 @@ DOCUMENTATION ------------- - Updated Value Node docs. +- Update manpage for Tools, and for the TOOL variable. diff --git a/SCons/Environment.py b/SCons/Environment.py index a1db8b6..4a1ae42 100644 --- a/SCons/Environment.py +++ b/SCons/Environment.py @@ -37,7 +37,7 @@ import re import shlex from collections import UserDict, UserList, deque from subprocess import PIPE, DEVNULL -from typing import Optional, Sequence +from typing import Callable, Collection, Optional, Sequence, Union import SCons.Action import SCons.Builder @@ -1250,9 +1250,11 @@ class Base(SubstitutionEnvironment): SCons.Tool.Initializers(self) if tools is None: - tools = self._dict.get('TOOLS', None) - if tools is None: - tools = ['default'] + tools = self._dict.get('TOOLS', ['default']) + else: + # for a new env, if we didn't use TOOLS, make sure it starts empty + # so it only shows tools actually initialized. + self._dict['TOOLS'] = [] apply_tools(self, tools, toolpath) # Now restore the passed-in and customized variables @@ -2014,11 +2016,20 @@ class Base(SubstitutionEnvironment): def _find_toolpath_dir(self, tp): return self.fs.Dir(self.subst(tp)).srcnode().get_abspath() - def Tool(self, tool, toolpath=None, **kwargs) -> SCons.Tool.Tool: + def Tool( + self, tool: Union[str, Callable], toolpath: Optional[Collection[str]] = None, **kwargs + ) -> Callable: """Find and run tool module *tool*. + *tool* is generally a string, but can also be a callable object, + in which case it is just called, without any of the setup. + The skipped setup includes storing *kwargs* into the created + :class:`~SCons.Tool.Tool` instance, which is extracted and used + when the instance is called, so in the skip case, the called + object will not get the *kwargs*. + .. versionchanged:: 4.2 - returns the tool module rather than ``None``. + returns the tool object rather than ``None``. """ if is_String(tool): tool = self.subst(tool) diff --git a/SCons/Environment.xml b/SCons/Environment.xml index 79b2503..c908d11 100644 --- a/SCons/Environment.xml +++ b/SCons/Environment.xml @@ -241,8 +241,11 @@ for more information). -A list of the names of the Tool specifications -that are part of this construction environment. +A list of the names of the Tool specification modules +that were actually initialized in the current &consenv;. +This may be useful as a diagnostic aid +to see if a tool did (or did not) run. +The value is informative and is not guaranteed to be complete. @@ -3437,32 +3440,43 @@ source_nodes = env.subst('$EXPAND_TO_NODELIST', conv=lambda x: x) Locates the tool specification module name and returns a callable tool object for that tool. -The tool module is searched for in standard locations -and in any paths specified by the optional -toolpath parameter. -The standard locations are &SCons;' own internal -path for tools plus the toolpath, if any (see the -Tools section in the manual page -for more details). -Any additional keyword arguments -kwargs are passed -to the tool module's generate function -during tool object construction. +When the environment method (&f-env-Tool;) form is used, +the tool object is automatically called before the method returns +to update env, +and name is +appended to the &cv-link-TOOLS; +&consvar; in that environment. +When the global function &f-Tool; form is used, +the tool object is constructed but not called, +as it lacks the context of an environment to update, +and the returned object needs to be used to arrange for the call. -When called, the tool object -updates a &consenv; with &consvars; and arranges +The tool module is searched for in the tool search paths (see the +Tools section in the manual page +for details) +and in any paths specified by the optional +toolpath parameter, +which must be a list of strings. +If toolpath is omitted, +the toolpath +supplied when the environment was created, +if any, is used. + + + +Any remaining keyword arguments are saved in +the tool object, +and will be passed to the tool module's +generate function +when the tool object is actually called. +The generate function +can update the &consenv; with &consvars; and arrange any other initialization -needed to use the mechanisms that tool describes. - - - -When the &f-env-Tool; form is used, -the tool object is automatically called to update env -and the value of tool is -appended to the &cv-link-TOOLS; -&consvar; in that environment. +needed to use the mechanisms that tool describes, +and can use these extra arguments to help +guide its actions. @@ -3481,10 +3495,7 @@ env.Tool('opengl', toolpath=['build/tools']) -When the global function &f-Tool; form is used, -the tool object is constructed but not called, -as it lacks the context of an environment to update. -The tool object can be passed to an +The returned tool object can be passed to an &f-link-Environment; or &f-link-Clone; call as part of the tools keyword argument, in which case the tool is applied to the environment being constructed, diff --git a/SCons/Tool/ToolTests.py b/SCons/Tool/ToolTests.py index e87a972..977730b 100644 --- a/SCons/Tool/ToolTests.py +++ b/SCons/Tool/ToolTests.py @@ -39,6 +39,7 @@ class DummyEnvironment: return progs[0] def Append(self, **kw) -> None: self.dict.update(kw) + AppendUnique = Append # wrong, but good enough for the use def __getitem__(self, key): return self.dict[key] def __setitem__(self, key, val) -> None: diff --git a/SCons/Tool/__init__.py b/SCons/Tool/__init__.py index 3681235..474414e 100644 --- a/SCons/Tool/__init__.py +++ b/SCons/Tool/__init__.py @@ -251,7 +251,7 @@ class Tool: kw.update(call_kw) else: kw = self.init_kw - env.Append(TOOLS=[self.name]) + env.AppendUnique(TOOLS=[self.name]) if hasattr(self, 'options'): import SCons.Variables if 'options' not in env: diff --git a/doc/man/scons.xml b/doc/man/scons.xml index a88fbfe..e9b3119 100644 --- a/doc/man/scons.xml +++ b/doc/man/scons.xml @@ -1749,15 +1749,18 @@ more useful. -Prevents the automatic addition of the standard -site_scons -directories to -sys.path. -Also prevents loading the -site_scons/site_init.py -modules if they exist, and prevents adding their -site_scons/site_tools -directories to the toolpath. +Do not read site directories. +Neither the standard site directories +(site_scons) +nor the path specified via a previous + option are +added to the module search path sys.path, +searched for a site_init.py file, +or have their site_tools +directory included in the tool search path. +Can be overridden by a subequent + option. + @@ -1880,7 +1883,7 @@ Also suppresses SCons status messages. -Use a specific path as the site directory +Use path as the site directory rather than searching the list of default site directories. This directory will be prepended to sys.path, @@ -1888,7 +1891,10 @@ the module path/site_init.py will be loaded if it exists, and path/site_tools -will be added to the default toolpath. +will be included in the tool search path. +The option is not additive - if given more than once, +the last path wins. + The default set of site directories searched when @@ -2672,7 +2678,7 @@ See for details. The optional variables keyword argument -allows passing a Variables object which will be used in the +allows passing a Variables object which will be used in the initialization of the &consenv; See for details. @@ -2683,75 +2689,83 @@ See for details. Tools -&SCons; has a large number of predefined tool modules +&SCons; has many included tool modules (more properly, tool specification modules) -which are used to help initialize the &consenv;. +which are used to help initialize the &consenv; prior to building, +and more can be written to suit a particular purpose, +or added from external sources (a repository of +constributed tools is available). +More information on writing custom tools can be found in the +Extending SCons section +and specifically Tool Modules. + + + An &SCons; tool is only responsible for setup. For example, if an &SConscript; file declares the need to construct an object file from a C-language source file by calling the -&b-link-Object; builder, then a tool representing +&b-link-Object; builder, then a tool module representing an available C compiler needs to have run first, to set up that builder and all the &consvars; -it needs in the associated &consenv;; the tool itself -is not called in the process of the build. Normally this -happens invisibly as &scons; has per-platform -lists of default tools, and it steps through those tools, -calling the ones which are actually applicable, -skipping those where necessary programs are not -installed on the build system, or other preconditions are not met. +it needs in the associated &consenv;. +The tool itself is not called in the process of the build. +Tool setup happens when a &consenv; is constructed, +and in the basic case needs no intervention - +platform-specific lists of default tools are used +to examine the specific capabilities of that platform and +configure the environment, +skipping those tools which are not applicable. -A specific set of tools -with which to initialize an environment when -creating it +If necessary, a specific set of tools to +initialize in an environment during creation may be specified using the optional keyword argument -tools, which takes a list -of tool names. +tools. +tools must be a list, +even if there are one (or zero) tools. This is useful to override the defaults, -to specify non-default built-in tools, and -to supply added tools: +to specify non-default built-in tools, +and to cause added tools to be called: + env = Environment(tools=['msvc', 'lex']) -Tools can also be directly called by using the &f-link-Tool; -method (see below). - - - The tools argument overrides the default tool list, it does not add to it, so be sure to include all the tools you need. -For example if you are building a c/c++ program +For example, if you are building a c/c++ program, you must specify a tool for at least a compiler and a linker, as in tools=['clang', 'link']. -The tool name 'default' can -be used to retain the default list. - - -If no tools argument is specified, -or if tools includes 'default', -then &scons; will auto-detect usable tools, -using the execution environment value of PATH -(that is, env['ENV']['PATH'] - -the external evironment &PATH; from os.environ -is not used) -for looking up any backing programs, and the platform name in effect + + + +If the tools argument is omitted, +or if tools includes +the reserved name 'default', +then &SCons; will auto-detect usable tools, +using the search path from the execution environment +(that is, env['ENV']['PATH']) +for looking up any external programs, +and the platform name in effect to determine the default tools for that platform. -Changing the PATH -variable after the &consenv; is constructed will not cause the tools to -be re-detected. +Note the contents of &PATH; from the external environment +os.environ is not used. +Changing the PATH in the execution environment +after the &consenv; is constructed will not cause the tools to +be re-detected. + -Additional tools can be added, see the -Extending SCons section -and specifically Tool Modules. + +Tools can also be directly called by using the &f-link-Tool; +method (see below). -SCons supports the following tool specifications out of the box: +&SCons; supports the following tool specifications out of the box: @@ -3674,18 +3688,44 @@ from SCons.Script import * -A &consenv; has an associated dictionary of -&consvars; -that are used by built-in or user-supplied build rules. + +&ConsVars; are key-value pairs +used to store information in a &consenv; that +is needed needed for builds using that environment. &Consvar; naming must follow the same rules as &Python; identifier naming: the initial character must be an underscore or letter, followed by any number of underscores, letters, or digits. The convention is to use uppercase for all letters for easier visual identification. -A &consenv; is not a &Python; dictionary itself, -but it can be indexed like one to access a -&consvar;: + + + +&Consvars; are used to hold many different types of information. +For example, the &cv-link-CPPDEFINES; variable is how to tell a C/C++ +compiler about preprocessor macros you need for your build. +The tool discovery that &SCons; performs will cause the +&cv-link-CXX; variable to hold the name of the C++ compiler, +if one was detected on the system, but you can give it a different +value to force a compiler command of a different name to be used. +Some variables contain lists of filename suffixes that are recognized +by a particular compiler chain. +&cv-link-BUILDERS; contains a mapping of configured +Builder names (e.g. &b-link-Textfile;) to the actual Builder instance +to call when that Builder is used. +&Consvars; may include references to other &consvars;: +the same tool which set up the C/C++ compiler will also set +up an "action string", describing how to invoke that compiler, +in &cv-link-CXXCOM;, which contains other &consvars; +using $VARIABLE syntax. +These references will be expanded and replaced on use +(see Variable Substitution). + + + +&Consvars; are referenced as if they were keys and values +in a &Python; dictionary: + env["CC"] = "cc" @@ -3702,8 +3742,15 @@ cvars = env.Dictionary() cvars["CC"] = "cc" -&Consvars; can also be passed to the &consenv; -constructor: + +in the previous example, since cvars +is an external copy, the value of &cv-CC; in the +&consenv; itself is not changed by the assignment. + + +&Consvars; can set by passing them as keyword arguments +when creating a new &consenv;: + env = Environment(CC="cc") @@ -3728,17 +3775,22 @@ This concept is called an override: env.Program('hello', 'hello.c', LIBS=['gl', 'glut']) -A number of useful &consvars; are automatically defined by -scons for each supported platform, and you can modify these -or define any additional &consvars; for your own use, -taking care not to overwrite ones which &SCons; is using. -The following is a list of the possible -automatically defined &consvars;. +Many useful &consvars; are automatically defined by +&SCons;, tuned to the specific platform in use, +and you can modify these or define any additional &consvars; +for use in your own Builders, Scanners and other tools. +Take care not to overwrite ones which &SCons; is using. +The following is a list of predefined &consvars;. +Pay attention to whether the values are ones +you may be expected to set vs. +ones that are set to expected values by +internal tools and other initializations +and probably should not be modified. Note the actual list available at execution time will never include all of these, as the ones -detected as not being useful (wrong platform, necessary +detected as not being applicable (wrong platform, necessary external command or files not installed, etc.) will not be set up. Correct build setups should be resilient to the possible absence of certain &consvars; before using them, @@ -6816,7 +6868,8 @@ see Special Attributes below). If expression refers to a &consvar;, -it is replaced with the value of that variable in the +it (including the $ or ${ }) +is replaced with the value of that variable in the &consenv; at the time of execution. If expression looks like a variable name but is not defined in the &consenv; @@ -7605,22 +7658,22 @@ env.Program('my_prog', ['file1.c', 'file2.f', 'file3.xyz']) Tool Modules -Additional tools can be added to a project either by -placing them in a site_tools subdirectory -of a site directory, or in a custom location specified to -&scons; by giving the +Custom tools can be added to a project either by +placing them in the site_tools subdirectory +of a configured site directory, +or in a location specified by the toolpath keyword argument to &f-link-Environment;. -A tool module is a form of &Python; module, invoked internally -using the &Python; import mechanism, so a tool can consist either -of a single source file taking the name of the tool -(e.g. mytool.py) or a directory taking -the name of the tool (e.g. mytool/) -which contains at least an __init__.py file. +You have to arrange to call a tool to put it into effect, +either as part of the list given to the tools +keyword argument at &consenv; initialization, +or by calling &f-link-env-Tool;. The toolpath parameter -takes a list as its value: +takes a list of path strings, +and the tools parameter +takes a list of tools, which are often strings: @@ -7628,23 +7681,23 @@ env = Environment(tools=['default', 'foo'], toolpath=['tools']) -This looks for a tool specification module (mytool.py, -or directory mytool) +This looks for a tool specification module foo in directory tools and in the standard locations, as well as using the ordinary default tools for the platform. -Directories specified via toolpath are prepended -to the existing tool path. -The default tool path is any site_tools directories, -so tools in a specified toolpath take priority, -followed by tools in a site_tools directory, -followed by built-in tools. For example, adding +When looking up tool specification modules, +directories specified via toolpath are +considered before the existing tool path +(site_tools subdirectories +of the default or specified site directories), +which are in turn considered before built-in tools. +For example, adding a tool specification module gcc.py to the toolpath directory would override the built-in &t-link-gcc; tool. -The tool path is stored in the environment and will be -used by subsequent calls to the &f-link-Tool; method, +The toolpath is saved in the environment +and will be used by subsequent calls to the &f-link-env-Tool; method, as well as by &f-link-env-Clone;. @@ -7655,7 +7708,14 @@ derived.CustomBuilder() -A tool specification module must include two functions: +A tool specification module is a form of &Python; module, +looked up internally using the &Python; import mechanism, +so a tool can consist either +of a single &Python; file taking the name of the tool +(e.g. mytool.py) or a directory taking +the name of the tool (e.g. mytool/) +which contains at least an __init__.py file. +A tool specification module has two required entry points: @@ -7665,16 +7725,17 @@ A tool specification module must include two functions: Modify the &consenv; env to set up necessary &consvars;, Builders, Emitters, etc., so the facilities represented by the tool can be executed. -Care should be taken not to overwrite &consvars; intended -to be settable by the user. For example: +Take Care not to overwrite &consvars; which may +have been explicitly set by the user, +retain and/or append instead. For example: def generate(env): ... if 'MYTOOL' not in env: env['MYTOOL'] = env.Detect("mytool") - if 'MYTOOLFLAGS' not in env: - env['MYTOOLFLAGS'] = SCons.Util.CLVar('--myarg') + flags = env.get('MYTOOLFLAGS', SCons.Util.CLVar()) + env.AppendUnique(MYTOOLFLAGS='--myarg') ... @@ -7687,9 +7748,9 @@ to vary its initialization. exists(env) -Return a true value if the tool can -be called in the context of env. -else false. +Return a truthy value if the tool can +be called in the context of env, +else return a falsy value. Usually this means looking up one or more known programs using the PATH from the supplied env, but the tool can @@ -7711,11 +7772,14 @@ and the exists function should still be provided. -The elements of the tools list may also -be functions or callable objects, -in which case the &Environment; method -will call those objects -to update the new &consenv; (see &f-link-Tool; for more details): +An element of the tools list may also +be a function or other callable object +(including a Tool object returned by a previous call to +&f-link-Tool;) in which case the &f-link-Environment; function +will directly call that object +to update the new &consenv;. +No tool lookup is done in this case. + def my_tool(env): @@ -7724,35 +7788,34 @@ def my_tool(env): env = Environment(tools=[my_tool]) -The individual elements of the tools list -may also themselves be lists or tuples of the form +An element of the tools list +may also be a two-element list or tuple of the form (toolname, kw_dict). -SCons searches for the +SCons searches for the a tool specification module toolname -specification file as described above, and -passes -kw_dict, -which must be a dictionary, as keyword arguments to the tool's -generate -function. -The -generate -function can use the arguments to modify the tool's behavior +as described above, +and passes kw_dict, +which must be a dictionary, +as keyword arguments to the tool's +generate function. +The generate +function can use those arguments to modify the tool's behavior by setting up the environment in different ways or otherwise changing its initialization. # in tools/my_tool.py: def generate(env, **kwargs): - # Sets MY_TOOL to the value of keyword 'arg1' '1' if not supplied - env['MY_TOOL'] = kwargs.get('arg1', '1') + # Sets MY_TOOL to the value of keyword 'arg1' or '1' if not supplied + env['MY_TOOL'] = kwargs.get('arg1', '1') def exists(env): - return True + return True # in SConstruct: -env = Environment(tools=['default', ('my_tool', {'arg1': 'abc'})], - toolpath=['tools']) +env = Environment( + tools=['default', ('my_tool', {'arg1': 'abc'})], toolpath=['tools'] +) The tool specification (my_tool in the example) diff --git a/test/site_scons/site-dir.py b/test/site_scons/site-dir.py index 97a3efe..9a00d91 100644 --- a/test/site_scons/site-dir.py +++ b/test/site_scons/site-dir.py @@ -81,7 +81,7 @@ scons: `.' is up to date.\n""", ) -# --site-dir followed by --no-site-dir turns processing off: +# --site-dir followed by --no-site-dir turns processing off test.run( arguments="-Q --site-dir=alt_site --no-site-dir .", stdout="""scons: `.' is up to date.\n""", @@ -95,6 +95,20 @@ test.run( stdout="""scons: `.' is up to date.\n""", ) +# no-site-dir followed by --site-dir picks up alt_site +test.run( + arguments='-Q --no-site-dir --site-dir=alt_site .', + stdout="""Hi there, I am in alt_site/site_init.py! +scons: `.' is up to date.\n""", +) + +# --site-dir is not additive +test.run( + arguments='-Q --site-dir=alt_site --site-dir=site_scons .', + stdout="""Hi there, I am in site_scons/site_init.py! +scons: `.' is up to date.\n""", +) + test.pass_test() -- cgit v0.12 From e696718d791dc77fac70de7837b43fd22678aa2f Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Mon, 1 Apr 2024 07:32:57 -0600 Subject: Fixes from review typo fix; more explanation of AppendUnique in DummyEnvironment test class. Signed-off-by: Mats Wichmann --- SCons/Tool/ToolTests.py | 4 +++- doc/man/scons.xml | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/SCons/Tool/ToolTests.py b/SCons/Tool/ToolTests.py index 977730b..9621b70 100644 --- a/SCons/Tool/ToolTests.py +++ b/SCons/Tool/ToolTests.py @@ -39,7 +39,9 @@ class DummyEnvironment: return progs[0] def Append(self, **kw) -> None: self.dict.update(kw) - AppendUnique = Append # wrong, but good enough for the use + # Adding a tool now calls AppendUnique so we need a mocked one. Since + # the only usage is adding one tool, using Append is good enough. + AppendUnique = Append def __getitem__(self, key): return self.dict[key] def __setitem__(self, key, val) -> None: diff --git a/doc/man/scons.xml b/doc/man/scons.xml index e9b3119..0ac9916 100644 --- a/doc/man/scons.xml +++ b/doc/man/scons.xml @@ -7725,8 +7725,8 @@ A tool specification module has two required entry points: Modify the &consenv; env to set up necessary &consvars;, Builders, Emitters, etc., so the facilities represented by the tool can be executed. -Take Care not to overwrite &consvars; which may -have been explicitly set by the user, +Take care not to overwrite &consvars; which may +have been explicitly set by the user; retain and/or append instead. For example: -- cgit v0.12