Support using a single .sconsign file. (Stephen Kennedy)

6 files changed, 268 insertions, 90 deletions

diff --git a/src/CHANGES.txt b/src/CHANGES.txt
index 7dcdaf9..1eeb7a9 100644
--- a/src/CHANGES.txt
+++ b/src/CHANGES.txt
@@ -20,6 +20,12 @@ RELEASE X.XX - XXX
   - Add support for CCVERSION and CXXVERSION variables for a number
     of C and C++ compilers.
 
+  From Stephen Kennedy:
+
+  - Add support for a configurable global .sconsign.dbm file which
+    can be used to avoid cluttering each directory with an individual
+    .sconsign file.
+
   From Steven Knight:
 
   - The -Q option suppressed too many messages; fix it so that it only
diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py
index 249c8c6..a78a2c9 100644
--- a/src/engine/SCons/Node/FS.py
+++ b/src/engine/SCons/Node/FS.py
@@ -1100,7 +1100,7 @@ class Dir(Base):
         creating it first if necessary."""
         if not self._sconsign:
             import SCons.Sig
-            self._sconsign = SCons.Sig.SConsignFile(self)
+            self._sconsign = SCons.Sig.SConsignForDirectory(self)
         return self._sconsign
 
     def srcnode(self):
diff --git a/src/engine/SCons/Script/SConscript.py b/src/engine/SCons/Script/SConscript.py
index 0f9daa7..1c17ee2 100644
--- a/src/engine/SCons/Script/SConscript.py
+++ b/src/engine/SCons/Script/SConscript.py
@@ -41,7 +41,6 @@ import SCons.Node.Python
 import SCons.Platform
 import SCons.SConf
 import SCons.Script
-import SCons.Tool
 import SCons.Util
 import SCons.Options
 import SCons
@@ -539,6 +538,13 @@ def SetOption(name, value):
 def GetOption(name):
     return SCons.Script.ssoptions.get(name)
 
+def SConsignFile(name=".sconsign.dbm"):
+    import SCons.Sig
+    if not os.path.isabs(name):
+        sd = str(SCons.Node.FS.default_fs.SConstruct_dir)
+        name = os.path.join(sd, name)
+    SCons.Sig.SConsignFile(name)
+
 def BuildDefaultGlobals():
     """
     Create a dictionary containing all the default globals for 
@@ -583,6 +589,7 @@ def BuildDefaultGlobals():
     globals['Return']            = Return
     globals['SConscript']        = SConscript
     globals['SConscriptChdir']   = SConscriptChdir
+    globals['SConsignFile']      = SConsignFile
     globals['Scanner']           = SCons.Scanner.Base
     globals['SetBuildSignatureType'] = SetBuildSignatureType
     globals['SetCommandHandler'] = SCons.Action.SetCommandHandler
diff --git a/src/engine/SCons/Sig/SigTests.py b/src/engine/SCons/Sig/SigTests.py
index af2ebab..c82d2c5 100644
--- a/src/engine/SCons/Sig/SigTests.py
+++ b/src/engine/SCons/Sig/SigTests.py
@@ -424,7 +424,7 @@ class _SConsignTestCase(unittest.TestCase):
         assert f.get('foo') == (3, 1, 2)
         assert f.get_implicit('foo') == ['bar']
 
-        f = SCons.Sig._SConsign(None, DummyModule())
+        f = SCons.Sig._SConsign(DummyModule())
         f.set_bsig('foo', 1)
         assert f.get('foo') == (None, 1, None)
         f.set_csig('foo', 2)
@@ -435,7 +435,38 @@ class _SConsignTestCase(unittest.TestCase):
         assert f.get('foo') == (3, 1, 2)
         assert f.get_implicit('foo') == ['bar']
 
-class SConsignFileTestCase(unittest.TestCase):
+class SConsignDBTestCase(unittest.TestCase):
+
+    def runTest(self):
+        class DummyNode:
+            def __init__(self, path):
+                self.path = path
+        save_SConsign_db = SCons.Sig.SConsign_db
+        SCons.Sig.SConsign_db = {}
+        try:
+            d1 = SCons.Sig.SConsignDB(DummyNode('dir1'))
+            d1.set_timestamp('foo', 1)
+            d1.set_bsig('foo', 2)
+            d1.set_csig('foo', 3)
+            d1.set_timestamp('bar', 4)
+            d1.set_bsig('bar', 5)
+            d1.set_csig('bar', 6)
+            assert d1.get('foo') == (1, 2, 3)
+            assert d1.get('bar') == (4, 5, 6)
+
+            d2 = SCons.Sig.SConsignDB(DummyNode('dir1'))
+            d2.set_timestamp('foo', 7)
+            d2.set_bsig('foo', 8)
+            d2.set_csig('foo', 9)
+            d2.set_timestamp('bar', 10)
+            d2.set_bsig('bar', 11)
+            d2.set_csig('bar', 12)
+            assert d2.get('foo') == (7, 8, 9)
+            assert d2.get('bar') == (10, 11, 12)
+        finally:
+            SCons.Sig.SConsign_db = save_SConsign_db
+
+class SConsignDirFileTestCase(unittest.TestCase):
 
     def runTest(self):
         class DummyModule:
@@ -448,7 +479,7 @@ class SConsignFileTestCase(unittest.TestCase):
         class DummyNode:
             path = 'not_a_valid_path'
 
-        f = SCons.Sig.SConsignFile(DummyNode(), DummyModule())
+        f = SCons.Sig.SConsignDirFile(DummyNode(), DummyModule())
         f.set_bsig('foo', 1)
         assert f.get('foo') == (None, 1, None)
         f.set_csig('foo', 2)
@@ -467,7 +498,8 @@ def suite():
     suite.addTest(CalcTestCase())
     suite.addTest(SConsignEntryTestCase())
     suite.addTest(_SConsignTestCase())
-    suite.addTest(SConsignFileTestCase())
+    suite.addTest(SConsignDBTestCase())
+    suite.addTest(SConsignDirFileTestCase())
     return suite
 
 if __name__ == "__main__":
diff --git a/src/engine/SCons/Sig/__init__.py b/src/engine/SCons/Sig/__init__.py
index b4b70b4..298db66 100644
--- a/src/engine/SCons/Sig/__init__.py
+++ b/src/engine/SCons/Sig/__init__.py
@@ -49,6 +49,8 @@ default_max_drift = 2*24*60*60
 #XXX Get rid of the global array so this becomes re-entrant.
 sig_files = []
 
+SConsign_db = None
+
 # 1 means use build signature for derived source files
 # 0 means use content signature for derived source files
 build_signature = 1
@@ -77,10 +79,16 @@ class SConsignEntry:
     implicit = None
 
 class _SConsign:
-
-    def __init__(self, fp=None, module=None):
+    """
+    This is the controlling class for the signatures for the collection of
+    entries associated with a specific directory.  The actual directory
+    association will be maintained by a subclass that is specific to
+    the underlying storage method.  This class provides a common set of
+    methods for fetching and storing the individual bits of information
+    that make up signature entry.
+    """
+    def __init__(self, module=None):
         """
-        fp - file pointer to read entries from
         module - the signature module being used
         """
 
@@ -91,18 +99,6 @@ class _SConsign:
         self.entries = {}
         self.dirty = 0
 
-        if fp:
-            try:
-                self.entries = cPickle.load(fp)
-                if type(self.entries) is not type({}):
-                    self.entries = {}
-                    raise TypeError
-            except:
-                SCons.Warnings.warn(SCons.Warnings.CorruptSConsignWarning,
-                                    "Ignoring corrupt .sconsign file: %s"%self.sconsign)
-        global sig_files
-        sig_files.append(self)
-
     # A null .sconsign entry.  We define this here so that it will
     # be easy to keep this in sync if/whenever we change the type of
     # information returned by the get() method, below.
@@ -185,11 +181,58 @@ class _SConsign:
         entry.implicit = implicit
         self.set_entry(filename, entry)
 
-class SConsignFile(_SConsign):
+class SConsignDB(_SConsign):
     """
-    Encapsulates reading and writing a .sconsign file.
+    A _SConsign subclass that reads and writes signature information
+    from a global .sconsign.dbm file.
     """
+    def __init__(self, dir, module=None):
+        _SConsign.__init__(self, module)
+
+        self.dir = dir
+
+        try:
+            global SConsign_db
+            rawentries = SConsign_db[self.dir.path]
+        except KeyError:
+            pass
+        else:
+            try:
+                self.entries = cPickle.loads(rawentries)
+                if type(self.entries) is not type({}):
+                    self.entries = {}
+                    raise TypeError
+            except:
+                SCons.Warnings.warn(SCons.Warnings.CorruptSConsignWarning,
+                                    "Ignoring corrupt sconsign entry : %s"%self.dir.path)
+
+        global sig_files
+        sig_files.append(self)
+
+    def write(self):
+        if self.dirty:
+            global SConsign_db
+            SConsign_db[self.dir.path] = cPickle.dumps(self.entries, 1)
+            SConsign_db.sync()
 
+class SConsignDir(_SConsign):
+    def __init__(self, fp=None, module=None):
+        """
+        fp - file pointer to read entries from
+        module - the signature module being used
+        """
+        _SConsign.__init__(self, module)
+
+        if fp:
+            self.entries = cPickle.load(fp)
+            if type(self.entries) is not type({}):
+                self.entries = {}
+                raise TypeError
+
+class SConsignDirFile(SConsignDir):
+    """
+    Encapsulates reading and writing a per-directory .sconsign file.
+    """
     def __init__(self, dir, module=None):
         """
         dir - the directory for the file
@@ -204,7 +247,14 @@ class SConsignFile(_SConsign):
         except:
             fp = None
 
-        _SConsign.__init__(self, fp, module)
+        try:
+            SConsignDir.__init__(self, fp, module)
+        except:
+            SCons.Warnings.warn(SCons.Warnings.CorruptSConsignWarning,
+                                "Ignoring corrupt .sconsign file: %s"%self.sconsign)
+
+        global sig_files
+        sig_files.append(self)
 
     def write(self):
         """
@@ -249,6 +299,21 @@ class SConsignFile(_SConsign):
             except:
                 pass
 
+SConsignForDirectory = SConsignDirFile
+
+def SConsignFile(name):
+    """
+    Arrange for all signatures to be stored in a global .sconsign.dbm
+    file.
+    """
+    global SConsign_db
+    if SConsign_db is None:
+        import anydbm
+        SConsign_db = anydbm.open(name, "c")
+
+    global SConsignForDirectory
+    SConsignForDirectory = SConsignDB
+
 class Calculator:
     """
     Encapsulates signature calculations and .sconsign file generating
diff --git a/src/script/sconsign.py b/src/script/sconsign.py
index d70f02b..015f1db 100644
--- a/src/script/sconsign.py
+++ b/src/script/sconsign.py
@@ -134,6 +134,108 @@ sys.path = libs + sys.path
 # END STANDARD SCons SCRIPT HEADER
 ##############################################################################
 
+PF_bsig      = 0x1
+PF_csig      = 0x2
+PF_timestamp = 0x4
+PF_implicit  = 0x8
+PF_all       = PF_bsig | PF_csig | PF_timestamp | PF_implicit
+
+Do_Func = None
+Print_Directories = []
+Print_Entries = []
+Print_Flags = 0
+Verbose = 0
+Readable = 0
+
+def field(name, pf, val):
+    if Print_Flags & pf:
+        if Verbose:
+            sep = "\n    " + name + ": "
+        else:
+            sep = " "
+        return sep + str(val)
+    else:
+        return ""
+
+def printfield(name, entry):
+    if Readable and entry.timestamp:
+        ts = "'" + time.ctime(entry.timestamp) + "'"
+    else:
+        ts = entry.timestamp
+    timestamp = field("timestamp", PF_timestamp, ts)
+    bsig = field("bsig", PF_bsig, entry.bsig)
+    csig = field("csig", PF_csig, entry.csig)
+    print name + ":" + timestamp + bsig + csig
+    if Print_Flags & PF_implicit and entry.implicit:
+        if Verbose:
+            print "    implicit:"
+        for i in entry.implicit:
+            print "        %s" % i
+
+def printentries(entries):
+    if Print_Entries:
+        for name in Print_Entries:
+            try:
+                entry = entries[name]
+            except KeyError:
+                sys.stderr.write("sconsign: no entry `%s' in `%s'\n" % (name, args[0]))
+            else:
+                printfield(name, entry)
+    else:
+        for name, e in entries.items():
+            printfield(name, e)
+
+import SCons.Sig
+
+def Do_SConsignDB(name):
+    import anydbm
+    import cPickle
+    try:
+        open(name, 'rb')
+    except (IOError, OSError), e:
+        sys.stderr.write("sconsign: %s\n" % (e))
+        return
+    try:
+        db = anydbm.open(name, "r")
+    except anydbm.error, e:
+        sys.stderr.write("sconsign: ignoring invalid .sconsign.dbm file `%s': %s\n" % (name, e))
+        return
+    if Print_Directories:
+        for dir in Print_Directories:
+            try:
+                val = db[dir]
+            except KeyError:
+                sys.stderr.write("sconsign: no dir `%s' in `%s'\n" % (dir, args[0]))
+            else:
+                entries = cPickle.loads(val)
+                print '=== ' + dir + ':'
+                printentries(entries)
+    else:
+        keys = db.keys()
+        keys.sort()
+        for dir in keys:
+            entries = cPickle.loads(db[dir])
+            print '=== ' + dir + ':'
+            printentries(entries)
+
+def Do_SConsignDir(name):
+    try:
+        fp = open(name, 'rb')
+    except (IOError, OSError), e:
+        sys.stderr.write("sconsign: %s\n" % (e))
+        return
+    try:
+        sconsign = SCons.Sig.SConsignDir(fp)
+    except:
+        sys.stderr.write("sconsign: ignoring invalid .sconsign file `%s'\n" % name)
+        return
+    printentries(sconsign.entries)
+
+Function_Map = {'dbm'      : Do_SConsignDB,
+                'sconsign' : Do_SConsignDir}
+
+##############################################################################
+
 import getopt
 
 helpstr = """\
@@ -141,7 +243,9 @@ Usage: sconsign [OPTIONS] FILE [...]
 Options:
   -b, --bsig                  Print build signature information.
   -c, --csig                  Print content signature information.
-  -e, --entry ENTRY           Print only info about ENTRY.
+  -d DIR, --dir=DIR           Print only info about DIR.
+  -e ENTRY, --entry=ENTRY     Print only info about ENTRY.
+  -f FORMAT, --format=FORMAT  FILE is in the specified FORMAT.
   -h, --help                  Print this message and exit.
   -i, --implicit              Print implicit dependency information.
   -r, --readable              Print timestamps in human-readable form.
@@ -149,84 +253,48 @@ Options:
   -v, --verbose               Verbose, describe each field.
 """
 
-opts, args = getopt.getopt(sys.argv[1:], "bce:hirtv",
-                            ['bsig', 'csig', 'entry=', 'help', 'implicit',
+opts, args = getopt.getopt(sys.argv[1:], "bcd:e:f:hirtv",
+                            ['bsig', 'csig', 'dir=', 'entry=',
+                             'format=', 'help', 'implicit',
                              'readable', 'timestamp', 'verbose'])
 
-pf_bsig      = 0x1
-pf_csig      = 0x2
-pf_timestamp = 0x4
-pf_implicit  = 0x8
-pf_all       = pf_bsig | pf_csig | pf_timestamp | pf_implicit
-
-entries = []
-printflags = 0
-verbose = 0
-readable = 0
-
 for o, a in opts:
     if o in ('-b', '--bsig'):
-        printflags = printflags | pf_bsig
+        Print_Flags = Print_Flags | PF_bsig
     elif o in ('-c', '--csig'):
-        printflags = printflags | pf_csig
+        Print_Flags = Print_Flags | PF_csig
+    elif o in ('-d', '--dir'):
+        Print_Directories.append(a)
     elif o in ('-e', '--entry'):
-        entries.append(a)
+        Print_Entries.append(a)
+    elif o in ('-f', '--format'):
+        try:
+            Do_Func = Function_Map[a]
+        except KeyError:
+            sys.stderr.write("sconsign: illegal file format `%s'\n" % a)
+            print helpstr
+            sys.exit(2)
     elif o in ('-h', '--help'):
         print helpstr
         sys.exit(0)
     elif o in ('-i', '--implicit'):
-        printflags = printflags | pf_implicit
+        Print_Flags = Print_Flags | PF_implicit
     elif o in ('-r', '--readable'):
-        readable = 1
+        Readable = 1
     elif o in ('-t', '--timestamp'):
-        printflags = printflags | pf_timestamp
+        Print_Flags = Print_Flags | PF_timestamp
     elif o in ('-v', '--verbose'):
-        verbose = 1
+        Verbose = 1
 
-if printflags == 0:
-    printflags = pf_all
-
-def field(name, pf, val):
-    if printflags & pf:
-        if verbose:
-            sep = "\n    " + name + ": "
-        else:
-            sep = " "
-        return sep + str(val)
-    else:
-        return ""
-
-def printfield(name, entry):
-    if readable and entry.timestamp:
-        ts = "'" + time.ctime(entry.timestamp) + "'"
-    else:
-        ts = entry.timestamp
-    timestamp = field("timestamp", pf_timestamp, ts)
-    bsig = field("bsig", pf_bsig, entry.bsig)
-    csig = field("csig", pf_csig, entry.csig)
-    print name + ":" + timestamp + bsig + csig
-    if printflags & pf_implicit and entry.implicit:
-        if verbose:
-            print "    implicit:"
-        for i in entry.implicit:
-            print "        %s" % i
-
-import SCons.Sig
-
-def do_sconsign(fp):
-    sconsign = SCons.Sig._SConsign(fp)
-    if entries:
-        for name in entries:
-            try:
-                entry = sconsign.entries[name]
-            except KeyError:
-                sys.stderr.write("sconsign: no entry `%s' in `%s'\n" % (name, args[0]))
-            else:
-                printfield(name, entry)
-    else:
-        for name, e in sconsign.entries.items():
-            printfield(name, e)
-    
+if Print_Flags == 0:
+    Print_Flags = PF_all
     
 for a in args:
-    do_sconsign(open(a, 'rb'))
+    if Do_Func:
+        Do_Func(a)
+    elif a[-4:] == '.dbm':
+        Do_SConsignDB(a)
+    else:
+        Do_SConsignDir(a)
+
+sys.exit(0)