Merged in thosrtanner/scons-cache-name (pull request #302)

Change the cache to use 2-character directories rather than one. Also makes a lot more tests run on windows

6 files changed, 234 insertions, 11 deletions

diff --git a/src/Announce.txt b/src/Announce.txt
index c20d7df..0b58958 100644
--- a/src/Announce.txt
+++ b/src/Announce.txt
@@ -44,6 +44,12 @@ Notes:
    a scanner found in SCANNERS (but not for built-in scanners), but now
    the Install builder will not scan recursively regardless in order
    to optimize Install behaviour and bring orthogonality to previous behaviour.
+* SCons handles cache directories a bit differently/
+** Cache files are now stored in 256 subdirectories in the cache directory by 
+   default (this stresses NFS less). Existing cache directories will remain as
+   current, but SCons will prompt you to run scons-configure-cache which will
+   allow you to migrate to the new layout, or confirm you want to use the
+   existing layout.
 
 +=================================================================
 
@@ -56,6 +62,8 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER
   changes.
 
   Please note the following important changes since release 2.4.0:
+    - New external tool scons-configurecache which allows some configuration of
+      how files in the cache are controlled.
     - Fix to swig tool - pick-up 'swig', 'swig3.0' and 'swig2.0' (in order).
     - Fix to swig tool - respect env['SWIG'] provided by user.
     - Fix for Bug # 2791 - Setup.py fails unnecessarily under Jython.
diff --git a/src/CHANGES.txt b/src/CHANGES.txt
index 0831787..3d2cf1c 100644
--- a/src/CHANGES.txt
+++ b/src/CHANGES.txt
@@ -6,6 +6,15 @@
 
 RELEASE  VERSION/DATE TO BE FILLED IN LATER
 
+  From Tom Tanner:
+    - change cache to use 2 character subdirectories, rather than one character,
+      so as not to give huge directories for large caches, a situation which
+      causes issues for NFS.
+      For existing caches, you will need to run the scons-configure-cache.py
+      script to update them to the new format. You will get a warning every time
+      you build until you co this.
+    - Fix a bunch of unit tests on windows
+
   From Dirk Baechle:
     - Removed a lot of compatibility methods and workarounds
       for Python versions < 2.7, in order to prepare the work
diff --git a/src/engine/SCons/CacheDir.py b/src/engine/SCons/CacheDir.py
index 3b7d06f..1690674 100644
--- a/src/engine/SCons/CacheDir.py
+++ b/src/engine/SCons/CacheDir.py
@@ -27,7 +27,8 @@ __doc__ = """
 CacheDir support
 """
 
-import os.path
+import json
+import os
 import stat
 import sys
 
@@ -72,7 +73,8 @@ CacheRetrieve = SCons.Action.Action(CacheRetrieveFunc, CacheRetrieveString)
 CacheRetrieveSilent = SCons.Action.Action(CacheRetrieveFunc, None)
 
 def CachePushFunc(target, source, env):
-    if cache_readonly: return
+    if cache_readonly:
+        return
 
     t = target[0]
     if t.nocache:
@@ -125,6 +127,10 @@ def CachePushFunc(target, source, env):
 
 CachePush = SCons.Action.Action(CachePushFunc, None)
 
+# Nasty hack to cut down to one warning for each cachedir path that needs
+# upgrading.
+warned = dict()
+
 class CacheDir(object):
 
     def __init__(self, path):
@@ -133,11 +139,63 @@ class CacheDir(object):
         except ImportError:
             msg = "No hashlib or MD5 module available, CacheDir() not supported"
             SCons.Warnings.warn(SCons.Warnings.NoMD5ModuleWarning, msg)
-            self.path = None
-        else:
-            self.path = path
+            path = None
+        self.path = path
         self.current_cache_debug = None
         self.debugFP = None
+        self.config = dict()
+        if path is None:
+            return
+        # See if there's a config file in the cache directory. If there is,
+        # use it. If there isn't, and the directory exists and isn't empty,
+        # produce a warning. If the directory doesn't exist or is empty,
+        # write a config file.
+        config_file = os.path.join(path, 'config')
+        if not os.path.exists(config_file):
+            # A note: There is a race hazard here, if two processes start and
+            # attempt to create the cache directory at the same time. However,
+            # python doesn't really give you the option to do exclusive file
+            # creation (it doesn't even give you the option to error on opening
+            # an existing file for writing...). The ordering of events here
+            # as an attempt to alleviate this, on the basis that it's a pretty
+            # unlikely occurence (it'd require two builds with a brand new cache
+            # directory)
+            if os.path.isdir(path) and len(os.listdir(path)) != 0:
+                self.config['prefix_len'] = 1
+                # When building the project I was testing this on, the warning
+                # was output over 20 times. That seems excessive
+                global warned
+                if self.path not in warned:
+                    msg = "Please upgrade your cache by running " +\
+                          " scons-configure-cache.py " +  self.path
+                    SCons.Warnings.warn(SCons.Warnings.CacheVersionWarning, msg)
+                    warned[self.path] = True
+            else:
+                if not os.path.isdir(path):
+                    try:
+                        os.makedirs(path)
+                    except OSError:
+                        # If someone else is trying to create the directory at
+                        # the same time as me, bad things will happen
+                        msg = "Failed to create cache directory " + path
+                        raise SCons.Errors.EnvironmentError(msg)
+                        
+                self.config['prefix_len'] = 2
+                if not os.path.exists(config_file):
+                    try:
+                        with open(config_file, 'w') as config:
+                            json.dump(self.config, config)
+                    except:
+                        msg = "Failed to write cache configuration for " + path
+                        raise SCons.Errors.EnvironmentError(msg)
+        else:
+            try:
+                with open(config_file) as config:
+                    self.config = json.load(config)
+            except ValueError:
+                msg = "Failed to read cache configuration for " + path
+                raise SCons.Errors.EnvironmentError(msg)
+            
 
     def CacheDebug(self, fmt, target, cachefile):
         if cache_debug != self.current_cache_debug:
@@ -152,7 +210,7 @@ class CacheDir(object):
             self.debugFP.write(fmt % (target, os.path.split(cachefile)[1]))
 
     def is_enabled(self):
-        return (cache_enabled and not self.path is None)
+        return cache_enabled and not self.path is None
 
     def is_readonly(self):
         return cache_readonly
@@ -164,7 +222,7 @@ class CacheDir(object):
             return None, None
 
         sig = node.get_cachedir_bsig()
-        subdir = sig[0].upper()
+        subdir = sig[:self.config['prefix_len']].upper()
         dir = os.path.join(self.path, subdir)
         return dir, os.path.join(dir, sig)
 
diff --git a/src/engine/SCons/CacheDirTests.py b/src/engine/SCons/CacheDirTests.py
index 7ac97ef..82fba8f 100644
--- a/src/engine/SCons/CacheDirTests.py
+++ b/src/engine/SCons/CacheDirTests.py
@@ -83,6 +83,11 @@ class BaseTestCase(unittest.TestCase):
             #node.binfo.ninfo.bsig = bsig
         return node
 
+    def tearDown(self):
+        os.remove(os.path.join(self._CacheDir.path, 'config'))
+        os.rmdir(self._CacheDir.path)
+        # Should that be shutil.rmtree?
+
 class CacheDirTestCase(BaseTestCase):
     """
     Test calling CacheDir code directly.
@@ -98,10 +103,12 @@ class CacheDirTestCase(BaseTestCase):
         SCons.Util.MD5collect = my_collect
 
         try:
-            f5 = self.File("cd.f5", 'a_fake_bsig')
+            name = 'a_fake_bsig'
+            f5 = self.File("cd.f5", name)
             result = self._CacheDir.cachepath(f5)
-            dirname = os.path.join('cache', 'A')
-            filename = os.path.join(dirname, 'a_fake_bsig')
+            len = self._CacheDir.config['prefix_len']
+            dirname = os.path.join('cache', name.upper()[:len])
+            filename = os.path.join(dirname, name)
             assert result == (dirname, filename), result
         finally:
             SCons.Util.MD5collect = save_collect
diff --git a/src/engine/SCons/Warnings.py b/src/engine/SCons/Warnings.py
index 5c27825..2495b89 100644
--- a/src/engine/SCons/Warnings.py
+++ b/src/engine/SCons/Warnings.py
@@ -41,10 +41,12 @@ class WarningOnByDefault(Warning):
 
 
 # NOTE:  If you add a new warning class, add it to the man page, too!
-
 class TargetNotBuiltWarning(Warning): # Should go to OnByDefault
     pass
 
+class CacheVersionWarning(WarningOnByDefault):
+    pass
+
 class CacheWriteErrorWarning(Warning):
     pass
 
diff --git a/src/script/scons-configure-cache.py b/src/script/scons-configure-cache.py
new file mode 100644
index 0000000..c1b7d59
--- /dev/null
+++ b/src/script/scons-configure-cache.py
@@ -0,0 +1,139 @@
+#! /usr/bin/env python

+#

+# SCons - a Software Constructor

+#

+# __COPYRIGHT__

+#

+# Permission is hereby granted, free of charge, to any person obtaining

+# a copy of this software and associated documentation files (the

+# "Software"), to deal in the Software without restriction, including

+# without limitation the rights to use, copy, modify, merge, publish,

+# distribute, sublicense, and/or sell copies of the Software, and to

+# permit persons to whom the Software is furnished to do so, subject to

+# the following conditions:

+#

+# The above copyright notice and this permission notice shall be included

+# in all copies or substantial portions of the Software.

+#

+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY

+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE

+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND

+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE

+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION

+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION

+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

+

+__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"

+

+__version__ = "__VERSION__"

+

+__build__ = "__BUILD__"

+

+__buildsys__ = "__BUILDSYS__"

+

+__date__ = "__DATE__"

+

+__developer__ = "__DEVELOPER__"

+

+import argparse

+import glob

+import json

+import os

+

+def rearrange_cache_entries(current_prefix_len, new_prefix_len):

+    print 'Changing prefix length from', current_prefix_len, 'to', new_prefix_len

+    dirs = set()

+    old_dirs = set()

+    for file in glob.iglob(os.path.join('*', '*')):

+        name = os.path.basename(file)

+        dir = name[:current_prefix_len].upper()

+        if dir not in old_dirs:

+            print 'Migrating', dir

+            old_dirs.add(dir)

+        dir = name[:new_prefix_len].upper()

+        if dir not in dirs:

+            os.mkdir(dir)

+            dirs.add(dir)

+        os.rename(file, os.path.join(dir, name))

+

+    # Now delete the original directories

+    for dir in old_dirs:

+        os.rmdir(dir)

+

+# This dictionary should have one entry per entry in the cache config

+# Each entry should have the following:

+#   implicit - (optional) This is to allow adding a new config entry and also

+#              changing the behaviour of the system at the same time. This

+#              indicates the value the config entry would have had if it had been

+#              specified.

+#   default - The value the config entry should have if it wasn't previously

+#             specified

+#   command-line - parameters to pass to ArgumentParser.add_argument

+#   converter - (optional) Function to call if it's necessary to do some work

+#               if this configuration entry changes

+config_entries = {

+    'prefix_len' : { 

+        'implicit' : 1, 

+        'default' : 2 ,

+        'command-line' : {

+            'help' : 'Length of cache file name used as subdirectory prefix',

+            'metavar' : '<number>',

+            'type' : int

+            },

+        'converter' : rearrange_cache_entries

+    }

+}

+parser = argparse.ArgumentParser(

+    description = 'Modify the configuration of an scons cache directory',

+    epilog = '''

+             Unless you specify an option, it will not be changed (if it is

+             already set in the cache config), or changed to an appropriate

+             default (it it is not set).

+             '''

+    )

+

+parser.add_argument('cache-dir', help='Path to scons cache directory')

+for param in config_entries:

+    parser.add_argument('--' + param.replace('_', '-'), 

+                        **config_entries[param]['command-line'])

+parser.add_argument('--version', action='version', version='%(prog)s 1.0')

+

+# Get the command line as a dict without any of the unspecified entries.

+args = dict(filter(lambda x: x[1], vars(parser.parse_args()).items()))

+

+# It seems somewhat strange to me, but positional arguments don't get the -

+# in the name changed to _, whereas optional arguments do...

+os.chdir(args['cache-dir'])

+del args['cache-dir']

+

+if not os.path.exists('config'):

+    # Validate the only files in the directory are directories 0-9, a-f

+    expected = [ '{:X}'.format(x) for x in range(0, 16) ]

+    if not set(os.listdir('.')).issubset(expected):

+        raise RuntimeError("This doesn't look like a version 1 cache directory")

+    config = dict()

+else:

+    with open('config') as conf:

+        config = json.load(conf)

+

+# Find any keys that aren't currently set but should be

+for key in config_entries:

+    if key not in config:

+        if 'implicit' in config_entries[key]:

+            config[key] = config_entries[key]['implicit']

+        else:

+            config[key] = config_entries[key]['default']

+        if key not in args:

+            args[key] = config_entries[key]['default']

+

+#Now we go through each entry in args to see if it changes an existing config

+#setting.

+for key in args:

+    if args[key] != config[key]:        

+        if 'converter' in config_entries[key]:

+            config_entries[key]['converter'](config[key], args[key])

+        config[key] = args[key]

+

+# and write the updated config file

+with open('config', 'w') as conf:

+    json.dump(config, conf)
\ No newline at end of file