From b0e44d9dc69b224e75c2ce755ec1a0b7f5585e49 Mon Sep 17 00:00:00 2001 From: Thomas Tanner Date: Sat, 19 Mar 2016 10:59:38 +0000 Subject: Cleanup of code/comments and rename and rewrite of upgrade script --- doc/man/scons.xml | 6 +- src/CHANGES.txt | 10 +-- src/engine/SCons/CacheDir.py | 35 +++++----- src/engine/SCons/Warnings.py | 2 +- src/script/scons-configure-cache.py | 123 +++++++++++++++++++++++++++++++++++ src/script/scons-rename-cachedirs.py | 78 ---------------------- 6 files changed, 151 insertions(+), 103 deletions(-) create mode 100644 src/script/scons-configure-cache.py delete mode 100644 src/script/scons-rename-cachedirs.py diff --git a/doc/man/scons.xml b/doc/man/scons.xml index 59ac678..27cf8b7 100644 --- a/doc/man/scons.xml +++ b/doc/man/scons.xml @@ -1707,10 +1707,10 @@ specifies the type of warnings to be enabled or disabled: - --warn=cache-v1, --warn=no-cache-v1 + --warn=cache-version, --warn=no-cache-version -Enables or disables warnings about the cache directory being in the -original (v1) layout. +Enables or disables warnings about the cache directory not using +the latest configuration information CacheDir(). These warnings are enabled by default. diff --git a/src/CHANGES.txt b/src/CHANGES.txt index 52cff62..2057199 100644 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -7,10 +7,12 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER From Tom Tanner: - - change cache to use 2 character subdirectories, rather than one character. - For existing caches, you will need to run the scons-rename-cachedirs.py - script to update them to the new format. You will get a warning for this - every time you build. + - change cache to use 2 character subdirectories, rather than one character, + so as not to give huge directories for large caches, a situation which + causes issues for NFS. + For existing caches, you will need to run the scons-configure-cache.py + script to update them to the new format. You will get a warning every time + you build until you co this. - Fix a bunch of unit tests on windows From Dirk Baechle: diff --git a/src/engine/SCons/CacheDir.py b/src/engine/SCons/CacheDir.py index 5b60bae..a941986 100644 --- a/src/engine/SCons/CacheDir.py +++ b/src/engine/SCons/CacheDir.py @@ -148,11 +148,12 @@ class CacheDir(object): self.config = defaultdict() if path is None: return - # See if there's a config file in the cache directory + # See if there's a config file in the cache directory. If there is, + # use it. If there isn't, and the directory exists and isn't empty, + # produce a warning. If the directory doesn't exist or is empty, + # write a config file. config_file = os.path.join(path, 'config') if not os.path.exists(config_file): - # If the directory exists and is not empty, we're likely version 1. - # # A note: There is a race hazard here, if two processes start and # attempt to create the cache directory at the same time. However, # python doesn't really give you the option to do exclusive file @@ -163,24 +164,24 @@ class CacheDir(object): # directory) if os.path.isdir(path) and len(os.listdir(path)) != 0: self.config['prefix_len'] = 1 + # When building the project I was testing this on, the warning + # was output over 20 times. That seems excessive + global warned + if self.path not in warned: + msg = "Please upgrade your cache by running " +\ + " scons-upgrade-cache.py " + self.path + SCons.Warnings.warn(SCons.Warnings.CacheVersionWarning, msg) + warned[self.path] = True else: if not os.path.isdir(path): os.makedirs(path) self.config['prefix_len'] = 2 - if not os.path.exists(config_file): - with open(config_file, 'w') as config: - self.config = json.dump(self.config, config) - with open(config_file) as config: - self.config = json.load(config) - # When building the project I was testing this on, this was output - # over 20 times. That seems excessive - global warned - if self.config['prefix_len'] == 1 and self.path not in warned: - msg = "Please update your cache by going into " + self.path +\ - " and running scons-rename-cachedirs.py" - SCons.Warnings.warn(SCons.Warnings.CacheV1Warning, msg) - warned[self.path] = True - + if not os.path.exists(config_file): + with open(config_file, 'w') as config: + self.config = json.dump(self.config, config) + else: + with open(config_file) as config: + self.config = json.load(config) def CacheDebug(self, fmt, target, cachefile): if cache_debug != self.current_cache_debug: diff --git a/src/engine/SCons/Warnings.py b/src/engine/SCons/Warnings.py index 16319b9..2495b89 100644 --- a/src/engine/SCons/Warnings.py +++ b/src/engine/SCons/Warnings.py @@ -44,7 +44,7 @@ class WarningOnByDefault(Warning): class TargetNotBuiltWarning(Warning): # Should go to OnByDefault pass -class CacheV1Warning(WarningOnByDefault): +class CacheVersionWarning(WarningOnByDefault): pass class CacheWriteErrorWarning(Warning): diff --git a/src/script/scons-configure-cache.py b/src/script/scons-configure-cache.py new file mode 100644 index 0000000..ec3a940 --- /dev/null +++ b/src/script/scons-configure-cache.py @@ -0,0 +1,123 @@ +#! /usr/bin/env python +# +# SCons - a Software Constructor +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +__version__ = "__VERSION__" + +__build__ = "__BUILD__" + +__buildsys__ = "__BUILDSYS__" + +__date__ = "__DATE__" + +__developer__ = "__DEVELOPER__" + +import argparse +import glob +import json +import os + +def rearrange_cache_entries(current_prefix_len, new_prefix_len): + print 'Changing prefix length from', current_prefix_len, 'to', new_prefix_len + dirs = set() + old_dirs = set() + for file in glob.iglob(os.path.join('*', '*')): + name = os.path.basename(file) + dir = name[:current_prefix_len].upper() + if dir not in old_dirs: + print 'Migrating', dir + old_dirs.add(dir) + dir = name[:new_prefix_len].upper() + if dir not in dirs: + os.mkdir(dir) + dirs.add(dir) + os.rename(file, os.path.join(dir, name)) + + # Now delete the original directories + for dir in old_dirs: + os.rmdir(dir) + +parser = argparse.ArgumentParser( + description = 'Modify the configuration of an scons cache directory', + epilog = ''' + Unless you specify an option, it will not be changed (if it is + already set in the cache config), or changed to an appropriate + default (it it is not set). + ''' + ) + +parser.add_argument('cache-dir', help='Path to scons cache directory') +parser.add_argument('--prefix-len', + help='Length of cache file name used as subdirectory prefix', + metavar = '', + type=int) +parser.add_argument('--version', action='version', version='%(prog)s 1.0') + +# Get the command line as a dict without any of the unspecified entries. +args = dict(filter(lambda x: x[1], vars(parser.parse_args()).items())) + +# It seems somewhat strange to me, but positional arguments don't get the - +# in the name changed to _, whereas optional arguments do... +os.chdir(args['cache-dir']) +del args['cache-dir'] + +# If a value isn't currently configured, this contains the way it behaves +# currently (implied), and the way it should behave afer running this script +# (default) +# FIXME: I should use this to construct the parameter list and supply an +# upgrade function +implicit = { + 'prefix_len' : { 'implied' : 1, 'default' : 2 } +} + +if not os.path.exists('config'): + # Validate the only files in the directory are directories 0-9, a-f + expected = [ '{:X}'.format(x) for x in range(0, 16) ] + if not set(os.listdir('.')).issubset(expected): + raise RuntimeError("This doesn't look like a version 1 cache directory") + config = dict() +else: + with open('config') as conf: + config = json.load(conf) + +# Find any keys that aren't currently set but should be +for key in implicit: + if key not in config: + config[key] = implicit[key]['implied'] + if key not in args: + args[key] = implicit[key]['default'] + +#Now we go through each entry in args to see if it changes an existing config +#setting. +for key in args: + if args[key] != config[key]: + if key == 'prefix_len': + rearrange_cache_entries(config[key], args[key]) + config[key] = args[key] + +# and write the updated config file +with open('config', 'w') as conf: + json.dump(config, conf) \ No newline at end of file diff --git a/src/script/scons-rename-cachedirs.py b/src/script/scons-rename-cachedirs.py deleted file mode 100644 index 24897d1..0000000 --- a/src/script/scons-rename-cachedirs.py +++ /dev/null @@ -1,78 +0,0 @@ -#! /usr/bin/env python -# -# SCons - a Software Constructor -# -# __COPYRIGHT__ -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - -__version__ = "__VERSION__" - -__build__ = "__BUILD__" - -__buildsys__ = "__BUILDSYS__" - -__date__ = "__DATE__" - -__developer__ = "__DEVELOPER__" - -import glob -import json -import os - -# The entire purpose of this script is to rename the files in the specified -# cache directory from the 16 single hex digit directories to 256 2 hex digit -# directories. - -# You run this in the cache directory. - -expected = ['{:X}'.format(x) for x in range(0, 16)] - -if not os.path.exists('config'): - # check there are 16 directories, 0 - 9, A - F - if sorted(glob.glob('*')) != expected: - raise RuntimeError("This doesn't look like a (version 1) cache directory") - config = { 'prefix_len' : 1 } -else: - with open('config') as conf: - config = json.load(conf) - if config['prefix_len'] != 1: - raise RuntimeError("This doesn't look like a (version 1) cache directory") - -dirs = set() -for file in glob.iglob(os.path.join('*', '*')): - name = os.path.basename(file) - dir = name[:2].upper() - if dir not in dirs: - os.mkdir(dir) - dirs.add(dir) - os.rename(file, os.path.join(dir, name)) - -# Now delete the original directories -for dir in expected: - if os.path.exists(dir): - os.rmdir(dir) - -# and write a config file -config['prefix_len'] = 2 -with open('config', 'w') as conf: - json.dump(config, conf) \ No newline at end of file -- cgit v0.12