From 0ebe7e7f4ac5ef86a48c0d016ab210c1e0a32549 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Wed, 19 Jul 2023 14:00:52 -0600 Subject: Tweak CacheDir docs [skip appveyor] Formatting, some wordsmithing. Signed-off-by: Mats Wichmann --- CHANGES.txt | 1 + RELEASE.txt | 1 + SCons/Environment.xml | 46 ++++++++++++++++++------------------ doc/user/caching.xml | 64 ++++++++++++++++++--------------------------------- 4 files changed, 49 insertions(+), 63 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 3725fa4..1c905b4 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -124,6 +124,7 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Cleaned up dblite module (checker warnings, etc.). - Some cleanup in the FortranCommon tool. - Changed the message about scons -H to clarify it shows built-in options. + - Documentation tweaks around CacheDir. From Jonathon Reinhart: - Fix another instance of `int main()` in CheckLib() causing failures diff --git a/RELEASE.txt b/RELEASE.txt index bffe740..663fb0f 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -106,6 +106,7 @@ DOCUMENTATION the results would be incorrect (does not apply to positional argument usage, which had no problem). - Changed the message about scons -H to clarify it shows built-in options only. +- Cachedir description updated. DEVELOPMENT ----------- diff --git a/SCons/Environment.xml b/SCons/Environment.xml index 2b4c4af..3b00f47 100644 --- a/SCons/Environment.xml +++ b/SCons/Environment.xml @@ -251,8 +251,8 @@ that are part of this construction environment. The class type that SCons should use when instantiating a -new &f-link-CacheDir; for the given environment. It must be -a subclass of the SCons.CacheDir.CacheDir class. +new &f-link-CacheDir; in this &consenv;. Must be +a subclass of the SCons.CacheDir.CacheDir class. @@ -883,16 +883,7 @@ disables derived file caching. -When specifying a -custom_class which should be a class type which is a subclass of -SCons.CacheDir.CacheDir, SCons will -internally invoke this class to use for performing caching operations. -This argument is optional and if left to default None, will use the -default SCons.CacheDir.CacheDir class. - - - -Calling the environment method +Calling the environment method form &f-link-env-CacheDir; limits the effect to targets built through the specified construction environment. @@ -906,6 +897,18 @@ caching by calling &f-env-CacheDir;. +Caching behavior can be configured by passing a specialized cache +class as the optional custom_class parameter. +This class must be a subclass of +SCons.CacheDir.CacheDir. +&SCons; will internally invoke the custom class for performing +caching operations. +If the parameter is omitted or set to +None, &SCons; will use the default +SCons.CacheDir.CacheDir class. + + + When derived-file caching is being used and &scons; @@ -930,15 +933,14 @@ identified by its &buildsig;, for future use. The Retrieved `file' from cache messages are useful for human consumption, -but less so when comparing log files between +but less useful when comparing log files between &scons; runs which will show differences that are noisy and not actually significant. To disable, use the option. -With this option, &scons; -will print the action that would -have been used to build the file without -considering cache retrieval. +With this option, &scons; changes printing +to always show the action that would +have been used to build the file without caching. @@ -946,10 +948,10 @@ Derived-file caching may be disabled for any invocation of &scons; by giving the -command line option. -Cache updating may be disabled, leaving cache +command line option; +cache updating may be disabled, leaving cache fetching enabled, by giving the -. + option. @@ -959,7 +961,7 @@ option is used, &scons; will place a copy of all -derived files in the cache, +derived files into the cache, even if they already existed and were not built by this invocation. This is useful to populate a cache @@ -984,7 +986,7 @@ predict or prohibitively large. Note that (at this time) &SCons; provides no facilities for managing the derived-file cache. It is up to the developer -to arrange for cache pruning, expiry, etc. if needed. +to arrange for cache pruning, expiry, access control, etc. if needed. diff --git a/doc/user/caching.xml b/doc/user/caching.xml index f00bd69..ff5d3a7 100644 --- a/doc/user/caching.xml +++ b/doc/user/caching.xml @@ -1,4 +1,10 @@ + + %scons; @@ -20,33 +26,6 @@ xsi:schemaLocation="http://www.scons.org/dbxsd/v1.0 http://www.scons.org/dbxsd/v1.0/scons.xsd"> Caching Built Files - - On multi-developer software projects, @@ -534,22 +513,25 @@ Program('prog', ['f1.c', 'f2.c', 'f3.c', 'f4.c', 'f5.c']) - &SCons;' internal CacheDir class can be extended to support customization - around the details of caching behaviors, for example using compressed cache files, - encrypted cache files, gathering statistics and data, or many other aspects. + You can customize the behavior of derived-file caching to + add your own features, for example to support compressed and/or + encrypted cache files, modify cache file permissions to better + support shared caches, gather additional statistics and data, etc. - To create your own custom cache class, - your custom class must be a subclass - of the SCons.CacheDir.CacheDir class. - You can then pass your custom class to the &f-link-CacheDir; - method or set the &consvar; - &cv-link-CACHEDIR_CLASS; to the class before configuring the cache - in that environment. - SCons will internally invoke and use your custom class when performing + To define custom cache behavior, subclass the + SCons.CacheDir.CacheDir class, + specializing those methods you want to change. + You can pass this custom class as the custom_class + paramter when calling &f-link-CacheDir; for global reach, + or when calling &f-link-env-CacheDir; for a specific environment. + You can also set the &consvar; + &cv-link-CACHEDIR_CLASS; to the custom class - this needs to happen + before configuring the cache in that environment. + &SCons; will internally invoke and use your custom class when performing cache operations. The below example shows a simple use case of overriding the copy_from_cache @@ -559,8 +541,8 @@ Program('prog', ['f1.c', 'f2.c', 'f3.c', 'f4.c', 'f5.c']) -import SCons import os +import SCons.CacheDir class CustomCacheDir(SCons.CacheDir.CacheDir): total_retrieved = 0 @@ -569,10 +551,10 @@ class CustomCacheDir(SCons.CacheDir.CacheDir): def copy_from_cache(cls, env, src, dst): # record total bytes pulled from cache cls.total_retrieved += os.stat(src).st_size - super().copy_from_cache(env, src, dst) + return super().copy_from_cache(env, src, dst) env = Environment() -env.CacheDir('scons-cache', CustomCacheDir) +env.CacheDir('scons-cache', custom_class=CustomCacheDir) # ... -- cgit v0.12 From c2fc9e2bbbceebc285b68b40fbbb88a95058fb3c Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Fri, 21 Jul 2023 16:18:54 -0600 Subject: Typo in doc/user/caching.xml [skip appveyor] Signed-off-by: Mats Wichmann --- doc/user/caching.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/user/caching.xml b/doc/user/caching.xml index ff5d3a7..8cca510 100644 --- a/doc/user/caching.xml +++ b/doc/user/caching.xml @@ -526,7 +526,7 @@ Program('prog', ['f1.c', 'f2.c', 'f3.c', 'f4.c', 'f5.c']) SCons.CacheDir.CacheDir class, specializing those methods you want to change. You can pass this custom class as the custom_class - paramter when calling &f-link-CacheDir; for global reach, + parameter when calling &f-link-CacheDir; for global reach, or when calling &f-link-env-CacheDir; for a specific environment. You can also set the &consvar; &cv-link-CACHEDIR_CLASS; to the custom class - this needs to happen -- cgit v0.12 From 06bdef751edd24dfcd6c7c557e536a46d65c370b Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sat, 22 Jul 2023 10:27:48 -0600 Subject: Incremental changes to runtest.py Keeping runtest.py changes relatively small, this batch is separated from a larger potential change. Signed-off-by: Mats Wichmann --- runtest.py | 91 +++++++++++++++++++++++++++++++++++--------------------------- 1 file changed, 52 insertions(+), 39 deletions(-) diff --git a/runtest.py b/runtest.py index 07d9f33..1922ccf 100755 --- a/runtest.py +++ b/runtest.py @@ -6,9 +6,10 @@ The SCons test suite consists of: - * unit tests - *Tests.py files from the SCons/ dir + * unit tests - *Tests.py files from the SCons/ directory * end-to-end tests - *.py files in the test/ directory that - require the custom SCons framework from testing/ + require the custom SCons framework from + testing/framework. This script adds SCons/ and testing/ directories to PYTHONPATH, performs test discovery and processes tests according to options. @@ -26,28 +27,29 @@ from abc import ABC, abstractmethod from io import StringIO from pathlib import Path, PurePath, PureWindowsPath from queue import Queue +from typing import List, TextIO, Optional cwd = os.getcwd() - -debug = None -scons = None -catch_output = False -suppress_output = False - +debug: Optional[str] = None +scons: Optional[str] = None +catch_output: bool = False +suppress_output: bool = False script = PurePath(sys.argv[0]).name -usagestr = """\ -%(script)s [OPTIONS] [TEST ...] -""" % locals() - +usagestr = f"{script} [OPTIONS] [TEST ...]" epilogstr = """\ Environment Variables: PRESERVE, PRESERVE_{PASS,FAIL,NO_RESULT}: preserve test subdirs TESTCMD_VERBOSE: turn on verbosity in TestCommand\ """ +# this is currently expected to be global, maybe refactor later? +unittests: List[str] + parser = argparse.ArgumentParser( - usage=usagestr, epilog=epilogstr, allow_abbrev=False, - formatter_class=argparse.RawDescriptionHelpFormatter + usage=usagestr, + epilog=epilogstr, + allow_abbrev=False, + formatter_class=argparse.RawDescriptionHelpFormatter, ) # test selection options: @@ -78,7 +80,15 @@ parser.add_argument('-D', '--devmode', action='store_true', help="Run tests in Python's development mode (Py3.7+ only).") parser.add_argument('-e', '--external', action='store_true', help="Run the script in external mode (for external Tools)") -parser.add_argument('-j', '--jobs', metavar='JOBS', default=1, type=int, + +def posint(arg: str) -> int: + """Special positive-int type for argparse""" + num = int(arg) + if num < 0: + raise argparse.ArgumentTypeError("JOBS value must not be negative") + return num + +parser.add_argument('-j', '--jobs', metavar='JOBS', default=1, type=posint, help="Run tests in JOBS parallel jobs (0 for cpu_count).") parser.add_argument('-l', '--list', action='store_true', dest='list_only', help="List available tests and exit.") @@ -125,7 +135,7 @@ outctl.add_argument('-q', '--quiet', action='store_false', help="Don't print the test being executed.") outctl.add_argument('-s', '--short-progress', action='store_true', help="""Short progress, prints only the command line - and a progress percentage.""") + and a progress percentage, no results.""") outctl.add_argument('-t', '--time', action='store_true', dest='print_times', help="Print test execution time.") outctl.add_argument('--verbose', metavar='LEVEL', type=int, choices=range(1, 4), @@ -134,14 +144,18 @@ outctl.add_argument('--verbose', metavar='LEVEL', type=int, choices=range(1, 4), 2=print commands and non-zero output, 3=print commands and all output).""") # maybe add? -# outctl.add_argument('--version', action='version', version='%s 1.0' % script) +# outctl.add_argument('--version', action='version', version=f'{script} 1.0') logctl = parser.add_argument_group(description='Log control options:') logctl.add_argument('-o', '--output', metavar='LOG', help="Save console output to LOG.") -logctl.add_argument('--xml', metavar='XML', help="Save results to XML in SCons XML format.") +logctl.add_argument( + '--xml', + metavar='XML', + help="Save results to XML in SCons XML format (use - for stdout).", +) # process args and handle a few specific cases: -args = parser.parse_args() +args: argparse.Namespace = parser.parse_args() # we can't do this check with an argparse exclusive group, since those # only work with optional args, and the cmdline tests (args.testlist) @@ -159,38 +173,30 @@ if args.retry: if args.testlistfile: # args.testlistfile changes from a string to a pathlib Path object try: - p = Path(args.testlistfile) - # TODO simplify when Py3.5 dropped - if sys.version_info.major == 3 and sys.version_info.minor < 6: - args.testlistfile = p.resolve() - else: - args.testlistfile = p.resolve(strict=True) + ptest = Path(args.testlistfile) + args.testlistfile = ptest.resolve(strict=True) except FileNotFoundError: sys.stderr.write( parser.format_usage() - + 'error: -f/--file testlist file "%s" not found\n' % p + + f'error: -f/--file testlist file "{args.testlistfile}" not found\n' ) sys.exit(1) if args.excludelistfile: # args.excludelistfile changes from a string to a pathlib Path object try: - p = Path(args.excludelistfile) - # TODO simplify when Py3.5 dropped - if sys.version_info.major == 3 and sys.version_info.minor < 6: - args.excludelistfile = p.resolve() - else: - args.excludelistfile = p.resolve(strict=True) + pexcl = Path(args.excludelistfile) + args.excludelistfile = pexcl.resolve(strict=True) except FileNotFoundError: sys.stderr.write( parser.format_usage() - + 'error: --exclude-list file "%s" not found\n' % p + + f'error: --exclude-list file "{args.excludelistfile}" not found\n' ) sys.exit(1) if args.jobs == 0: try: - # on Linux, check available rather then physical CPUs + # on Linux, check available rather than physical CPUs args.jobs = len(os.sched_getaffinity(0)) except AttributeError: # Windows @@ -288,6 +294,7 @@ if sys.platform == 'win32': E_POINTER = ctypes.c_long(0x80004003).value def get_template_command(filetype, verb=None): + """Return the association-related string for *filetype*""" flags = ASSOCF_INIT_IGNOREUNKNOWN | ASSOCF_NOTRUNCATE assoc_str = ASSOCSTR_COMMAND cch = ctypes.c_ulong(260) @@ -427,7 +434,7 @@ class PopenExecutor(RuntestBase): # definition of spawn_it() above. if args.allow_pipe_files: - def execute(self, env): + def execute(self, env) -> None: # Create temporary files tmp_stdout = tempfile.TemporaryFile(mode='w+t') tmp_stderr = tempfile.TemporaryFile(mode='w+t') @@ -438,6 +445,7 @@ class PopenExecutor(RuntestBase): stderr=tmp_stderr, shell=False, env=env, + check=False, ) self.status = cp.returncode @@ -454,13 +462,14 @@ class PopenExecutor(RuntestBase): tmp_stderr.close() else: - def execute(self, env): + def execute(self, env) -> None: cp = subprocess.run( self.command_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False, env=env, + check=False, ) self.status, self.stdout, self.stderr = cp.returncode, cp.stdout, cp.stderr @@ -494,7 +503,11 @@ else: if not args.baseline or args.baseline == '.': baseline = cwd elif args.baseline == '-': - print("This logic used to checkout from svn. It's been removed. If you used this, please let us know on devel mailing list, IRC, or discord server") + sys.stderr.write( + "'baseline' logic used to checkout from svn. It has been removed. " + "If you used this, please let us know on devel mailing list, " + "IRC, or discord server\n" + ) sys.exit(-1) else: baseline = args.baseline @@ -601,7 +614,7 @@ def scanlist(testfile): def find_unit_tests(directory): """ Look for unit tests """ result = [] - for dirpath, dirnames, filenames in os.walk(directory): + for dirpath, _, filenames in os.walk(directory): # Skip folders containing a sconstest.skip file if 'sconstest.skip' in filenames: continue @@ -615,7 +628,7 @@ def find_unit_tests(directory): def find_e2e_tests(directory): """ Look for end-to-end tests """ result = [] - for dirpath, dirnames, filenames in os.walk(directory): + for dirpath, _, filenames in os.walk(directory): # Skip folders containing a sconstest.skip file if 'sconstest.skip' in filenames: continue -- cgit v0.12 From c7ef32e3d51e8dbacc7910fd89682450de0471e7 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 22 Jul 2023 15:45:36 -0700 Subject: minor edit --- SCons/Environment.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SCons/Environment.xml b/SCons/Environment.xml index 3b00f47..9d27c72 100644 --- a/SCons/Environment.xml +++ b/SCons/Environment.xml @@ -883,7 +883,7 @@ disables derived file caching. -Calling the environment method form +Calling the environment method &f-link-env-CacheDir; limits the effect to targets built through the specified construction environment. -- cgit v0.12