From a014c40490ca3353d82476ca6a1db2ad80ca57fe Mon Sep 17 00:00:00 2001 From: Thomas Tanner Date: Sat, 12 Mar 2016 23:28:47 +0000 Subject: improve behaviour --- src/engine/SCons/CacheDir.py | 43 ++++++++++++++++++++++++++++++------ src/engine/SCons/CacheDirTests.py | 13 ++++++++--- src/script/scons-rename-cachedirs.py | 30 +++++++++++++++++++------ test/CacheDir/CacheDir.py | 3 ++- test/CacheDir/environment.py | 3 ++- 5 files changed, 73 insertions(+), 19 deletions(-) diff --git a/src/engine/SCons/CacheDir.py b/src/engine/SCons/CacheDir.py index be0163d..728871f 100644 --- a/src/engine/SCons/CacheDir.py +++ b/src/engine/SCons/CacheDir.py @@ -27,7 +27,10 @@ __doc__ = """ CacheDir support """ -import os.path +from collections import defaultdict + +import json +import os import stat import sys @@ -72,7 +75,8 @@ CacheRetrieve = SCons.Action.Action(CacheRetrieveFunc, CacheRetrieveString) CacheRetrieveSilent = SCons.Action.Action(CacheRetrieveFunc, None) def CachePushFunc(target, source, env): - if cache_readonly: return + if cache_readonly: + return t = target[0] if t.nocache: @@ -133,11 +137,36 @@ class CacheDir(object): except ImportError: msg = "No hashlib or MD5 module available, CacheDir() not supported" SCons.Warnings.warn(SCons.Warnings.NoMD5ModuleWarning, msg) - self.path = None - else: - self.path = path + path = None + self.path = path self.current_cache_debug = None self.debugFP = None + self.config = defaultdict() + if path is None: + return + # See if there's a config file in the cache directory + config_file = os.path.join(path, 'config') + if not os.path.exists(config_file): + # If the directory exists we're likely version 1, otherwise + # assume we're latest. + # A note: There is a race hazard here, if two processes start and + # attempt to create the cache directory at the same time. However, + # python doesn't really give you the option to do exclusive file + # creation (it doesn't even give you the option to error on opening + # an existing file for writing...). The ordering of events here + # as an attempt to alleviate this, on the basis that it's a pretty + # unlikely occurence (it'd require two builds with a brand new cache + # directory) + if os.path.isdir(path): + self.config['prefix_len'] = 1 + else: + os.makedirs(path) + self.config['prefix_len'] = 2 + if not os.path.exists(config_file): + with open(config_file, 'w') as config: + self.config = json.dump(self.config, config) + with open(config_file) as config: + self.config = json.load(config) def CacheDebug(self, fmt, target, cachefile): if cache_debug != self.current_cache_debug: @@ -152,7 +181,7 @@ class CacheDir(object): self.debugFP.write(fmt % (target, os.path.split(cachefile)[1])) def is_enabled(self): - return (cache_enabled and not self.path is None) + return cache_enabled and not self.path is None def is_readonly(self): return cache_readonly @@ -164,7 +193,7 @@ class CacheDir(object): return None, None sig = node.get_cachedir_bsig() - subdir = sig[:2].upper() + subdir = sig[:self.config['prefix_len']].upper() dir = os.path.join(self.path, subdir) return dir, os.path.join(dir, sig) diff --git a/src/engine/SCons/CacheDirTests.py b/src/engine/SCons/CacheDirTests.py index 3b99d41..82fba8f 100644 --- a/src/engine/SCons/CacheDirTests.py +++ b/src/engine/SCons/CacheDirTests.py @@ -83,6 +83,11 @@ class BaseTestCase(unittest.TestCase): #node.binfo.ninfo.bsig = bsig return node + def tearDown(self): + os.remove(os.path.join(self._CacheDir.path, 'config')) + os.rmdir(self._CacheDir.path) + # Should that be shutil.rmtree? + class CacheDirTestCase(BaseTestCase): """ Test calling CacheDir code directly. @@ -98,10 +103,12 @@ class CacheDirTestCase(BaseTestCase): SCons.Util.MD5collect = my_collect try: - f5 = self.File("cd.f5", 'a_fake_bsig') + name = 'a_fake_bsig' + f5 = self.File("cd.f5", name) result = self._CacheDir.cachepath(f5) - dirname = os.path.join('cache', 'A_') - filename = os.path.join(dirname, 'a_fake_bsig') + len = self._CacheDir.config['prefix_len'] + dirname = os.path.join('cache', name.upper()[:len]) + filename = os.path.join(dirname, name) assert result == (dirname, filename), result finally: SCons.Util.MD5collect = save_collect diff --git a/src/script/scons-rename-cachedirs.py b/src/script/scons-rename-cachedirs.py index dd76a42..24897d1 100644 --- a/src/script/scons-rename-cachedirs.py +++ b/src/script/scons-rename-cachedirs.py @@ -36,6 +36,7 @@ __date__ = "__DATE__" __developer__ = "__DEVELOPER__" import glob +import json import os # The entire purpose of this script is to rename the files in the specified @@ -43,20 +44,35 @@ import os # directories. # You run this in the cache directory. + expected = ['{:X}'.format(x) for x in range(0, 16)] -# check there are 16 directories, 0 - 9, A - F -if sorted(glob.glob('*')) != [x]: - raise RuntimeError("This doesn't look like a cache directory") + +if not os.path.exists('config'): + # check there are 16 directories, 0 - 9, A - F + if sorted(glob.glob('*')) != expected: + raise RuntimeError("This doesn't look like a (version 1) cache directory") + config = { 'prefix_len' : 1 } +else: + with open('config') as conf: + config = json.load(conf) + if config['prefix_len'] != 1: + raise RuntimeError("This doesn't look like a (version 1) cache directory") + dirs = set() for file in glob.iglob(os.path.join('*', '*')): name = os.path.basename(file) dir = name[:2].upper() - print dir, name if dir not in dirs: os.mkdir(dir) dirs.add(dir) os.rename(file, os.path.join(dir, name)) - # Now delete the original directories - for dir in expected: - os.rmdir(dir) \ No newline at end of file +# Now delete the original directories +for dir in expected: + if os.path.exists(dir): + os.rmdir(dir) + +# and write a config file +config['prefix_len'] = 2 +with open('config', 'w') as conf: + json.dump(config, conf) \ No newline at end of file diff --git a/test/CacheDir/CacheDir.py b/test/CacheDir/CacheDir.py index 9abe0e0..4c05634 100644 --- a/test/CacheDir/CacheDir.py +++ b/test/CacheDir/CacheDir.py @@ -82,7 +82,8 @@ test.must_not_exist(src_aaa_out) test.must_not_exist(src_bbb_out) test.must_not_exist(src_ccc_out) test.must_not_exist(src_all) -test.fail_test(len(os.listdir(cache))) +# Even if you do -n, the cache will be configured. +test.fail_test(os.listdir(cache) != ['config']) # Verify that a normal build works correctly, and clean up. # This should populate the cache with our derived files. diff --git a/test/CacheDir/environment.py b/test/CacheDir/environment.py index 4fb9b51..1378bb2 100644 --- a/test/CacheDir/environment.py +++ b/test/CacheDir/environment.py @@ -85,7 +85,8 @@ test.must_not_exist(src_aaa_out) test.must_not_exist(src_bbb_out) test.must_not_exist(src_ccc_out) test.must_not_exist(src_all) -test.fail_test(len(os.listdir(cache))) +# Even if you do -n, the cache will be configured. +test.fail_test(os.listdir(cache) != ['config']) # Verify that a normal build works correctly, and clean up. # This should populate the cache with our derived files. -- cgit v0.12