Make SConsignFile() behavior the default.

9 files changed, 220 insertions, 69 deletions

diff --git a/src/CHANGES.txt b/src/CHANGES.txt
index cfd7097..5513057 100644
--- a/src/CHANGES.txt
+++ b/src/CHANGES.txt
@@ -266,6 +266,11 @@ RELEASE 0.97 - XXX
     that accomodate parser generators that write header files to a
     different suffix than the hard-coded .hpp when the -d option is used.
 
+  - The default behavior is now to store signature information in a
+    single .sconsign.dblite file in the top-level SConstruct directory.
+    The old behavior of a separate .sconsign file in each directory can
+    be specified by calling SConsignFile(None).
+
   From Wayne Lee:
 
   - Avoid "maximum recursion limit" errors when removing $(-$) pairs
diff --git a/src/RELEASE.txt b/src/RELEASE.txt
index 9c09e60..47b0031 100644
--- a/src/RELEASE.txt
+++ b/src/RELEASE.txt
@@ -58,7 +58,33 @@ RELEASE 0.97 - XXX
         or /usr/local was passed as the source to a Builder or Command()
         call, in which case SCons would scan the entire directory tree.
 
-    --  SIGNATURE CHANGES WILL CAUSE LIKELY REBUILDS AFTER UPGRADE
+    --  SIGNATURES ARE NOW STORED IN AN SConsignFile() BY DEFAULT,
+        CAUSING LIKELY REBUILDS; SPECIAL NOTE CONCERNING INTERACTION
+        WITH REPOSITORIES
+
+        The default behavior has been changed to store signature
+        information in a single .sconsign.dblite file in the top-level
+        SConstruct file.  This will cause rebuilds on upgrade to 0.97,
+        unless you were already calling the SConsignFile() function in
+        your SConscript files.
+
+        The previous default behavior was to store signature information
+        in a .sconsign file in each directory that contained target
+        files that SCons knew about.  The old behavior may be preserved
+        by specifying:
+
+              SConsignFile(None)
+
+        in any SConscript file.
+
+        If you are using the Repository feature, are not already using
+        the SConsignFile() function in your build, you *must* add
+        SConsignFile(None) to your build to keep interoperating with an
+        existing Repository that uses the old behavior of a .sconsign
+        file in each directory.  Alternatively, you can rebuild the
+        Repository with the new default behavior.
+
+    --  OTHER SIGNATURE CHANGES WILL CAUSE LIKELY REBUILDS AFTER UPGRADE
 
         This release adds several changes to the signature mechanism that
         will cause SCons to rebuild most configurations after upgrading
diff --git a/src/engine/SCons/Environment.py b/src/engine/SCons/Environment.py
index b813cf1..58440d6 100644
--- a/src/engine/SCons/Environment.py
+++ b/src/engine/SCons/Environment.py
@@ -1367,9 +1367,10 @@ class Base(SubstitutionEnvironment):
         return apply(SCons.Scanner.Scanner, nargs, nkw)
 
     def SConsignFile(self, name=".sconsign", dbm_module=None):
-        name = self.subst(name)
-        if not os.path.isabs(name):
-            name = os.path.join(str(self.fs.SConstruct_dir), name)
+        if not name is None:
+            name = self.subst(name)
+            if not os.path.isabs(name):
+                name = os.path.join(str(self.fs.SConstruct_dir), name)
         SCons.SConsign.File(name, dbm_module)
 
     def SideEffect(self, side_effect, target):
diff --git a/src/engine/SCons/EnvironmentTests.py b/src/engine/SCons/EnvironmentTests.py
index fcd0fee..495fab0 100644
--- a/src/engine/SCons/EnvironmentTests.py
+++ b/src/engine/SCons/EnvironmentTests.py
@@ -2568,6 +2568,10 @@ def generate(env):
             env.SConsignFile()
             assert fnames[5] == os.path.join(os.sep, 'dir', '.sconsign'), fnames
             assert dbms[5] == None, dbms
+
+            env.SConsignFile(None)
+            assert fnames[6] == None, fnames
+            assert dbms[6] == None, dbms
         finally:
             SCons.SConsign.File = save_SConsign_File
 
diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py
index 75efc15..9d7086c 100644
--- a/src/engine/SCons/Node/FS.py
+++ b/src/engine/SCons/Node/FS.py
@@ -422,6 +422,10 @@ class Base(SCons.Node.Node):
             self.path = name
         else:
             self.path = directory.entry_path(name)
+        if directory.tpath == '.':
+            self.tpath = name
+        else:
+            self.tpath = directory.entry_tpath(name)
         self.path_elements = directory.path_elements + [self]
 
         self.dir = directory
@@ -732,6 +736,7 @@ class FS(LocalFS):
 
         self.Top = self._doLookup(Dir, os.path.normpath(self.pathTop))
         self.Top.path = '.'
+        self.Top.tpath = '.'
         self._cwd = self.Top
 
     def clear_cache(self):
@@ -1102,6 +1107,7 @@ class Dir(Base):
     def addRepository(self, dir):
         if dir != self and not dir in self.repositories:
             self.repositories.append(dir)
+            dir.tpath = '.'
             self.__clearRepositoryCache()
 
     def up(self):
@@ -1284,6 +1290,9 @@ class Dir(Base):
     def entry_path(self, name):
         return self.path + os.sep + name
 
+    def entry_tpath(self, name):
+        return self.tpath + os.sep + name
+
     def must_be_a_Dir(self):
         """Called to make sure a Node is a Dir.  Since we're already
         one, this is a no-op for us."""
@@ -1389,6 +1398,7 @@ class RootDir(Dir):
         # won't gag won't it calls some of our methods.
         self.abspath = ''
         self.path = ''
+        self.tpath = ''
         self.path_elements = []
         self.duplicate = 0
         Base.__init__(self, name, self, fs)
@@ -1397,6 +1407,7 @@ class RootDir(Dir):
         # initial drive letter (the name) plus the directory separator.
         self.abspath = name + os.sep
         self.path = name + os.sep
+        self.tpath = name + os.sep
         self._morph()
 
     def __str__(self):
@@ -1408,6 +1419,9 @@ class RootDir(Dir):
     def entry_path(self, name):
         return self.path + name
 
+    def entry_tpath(self, name):
+        return self.tpath + name
+
     def is_under(self, dir):
         if self is dir:
             return 1
diff --git a/src/engine/SCons/SConfTests.py b/src/engine/SCons/SConfTests.py
index a5e0700..2685c2b 100644
--- a/src/engine/SCons/SConfTests.py
+++ b/src/engine/SCons/SConfTests.py
@@ -44,10 +44,15 @@ class SConfTestCase(unittest.TestCase):
 
     def setUp(self):
         # we always want to start with a clean directory
+        self.save_cwd = os.getcwd()
         self.test = TestCmd.TestCmd(workdir = '') 
+        os.chdir(self.test.workpath(''))
 
     def tearDown(self):
         self.test.cleanup()
+        import SCons.SConsign
+        SCons.SConsign.Reset()
+        os.chdir(self.save_cwd)
 
     def _resetSConfState(self):
         # Ok, this is tricky, and i do not know, if everything is sane.
@@ -104,7 +109,7 @@ class SConfTestCase(unittest.TestCase):
                                  log_file=self.test.workpath('config.log'))
         try:
             res = checks( self, sconf, TryFunc )
-            assert res[0] and not res[1] 
+            assert res[0] and not res[1], res
         finally:
             sconf.Finish()
             
@@ -115,7 +120,7 @@ class SConfTestCase(unittest.TestCase):
                                  log_file=self.test.workpath('config.log'))
         try:
             res = checks( self, sconf, TryFunc )
-            assert res[0] and not res[1] 
+            assert res[0] and not res[1], res
         finally:
             sconf.Finish()
         # we should have exactly one one error cached 
@@ -136,7 +141,7 @@ class SConfTestCase(unittest.TestCase):
         try:
             res = checks( self, sconf, TryFunc )
             log = self.test.read( self.test.workpath('config.log') )
-            assert res[0] and res[1] 
+            assert res[0] and res[1], res
         finally:
             sconf.Finish()
 
@@ -231,8 +236,8 @@ int main() {
                                  log_file=self.test.workpath('config.log'))
         try:
             res = checks(sconf)
-            assert res[0][0] and res[0][1] == "Hello" 
-            assert not res[1][0] and res[1][1] == ""
+            assert res[0][0] and res[0][1] == "Hello", res
+            assert not res[1][0] and res[1][1] == "", res
         finally:
             sconf.Finish()
         log = self.test.read( self.test.workpath('config.log') )
@@ -244,8 +249,8 @@ int main() {
                                  log_file=self.test.workpath('config.log'))
         try:
             res = checks(sconf)
-            assert res[0][0] and res[0][1] == "Hello" 
-            assert not res[1][0] and res[1][1] == ""
+            assert res[0][0] and res[0][1] == "Hello", res
+            assert not res[1][0] and res[1][1] == "", res
         finally:
             sconf.Finish()
         # we should have exactly one error cached 
@@ -271,9 +276,9 @@ int main() {
                                   log_file=self.test.workpath('config.log'))
         try:
             (ret, output) = sconf.TryAction(action=actionOK)
-            assert ret and output == "RUN OK"
+            assert ret and output == "RUN OK", (ret, output)
             (ret, output) = sconf.TryAction(action=actionFAIL)
-            assert not ret and output == ""
+            assert not ret and output == "", (ret, output)
         finally:
             sconf.Finish()
 
@@ -514,7 +519,7 @@ int main() {
 """
             (ret, output) = test.TryRun( prog, ".c" )
             test.Result( ret )
-            assert ret and output == "Hello" 
+            assert ret and output == "Hello", (ret, output)
             return ret
         
 
@@ -525,7 +530,7 @@ int main() {
                                  log_file=self.test.workpath('config.log'))
         try:
             ret = sconf.CheckCustom()
-            assert ret 
+            assert ret, ret
         finally:
             sconf.Finish()
             
diff --git a/src/engine/SCons/SConsign.py b/src/engine/SCons/SConsign.py
index 212ec8d..9b5c420 100644
--- a/src/engine/SCons/SConsign.py
+++ b/src/engine/SCons/SConsign.py
@@ -35,15 +35,68 @@ import os.path
 import string
 import time
 
+import SCons.dblite
 import SCons.Node
 import SCons.Sig
 import SCons.Warnings
 
+from SCons.Debug import Trace
+
+def corrupt_dblite_warning(filename):
+    SCons.Warnings.warn(SCons.Warnings.CorruptSConsignWarning,
+                        "Ignoring corrupt .sconsign file: %s"%filename)
+
+SCons.dblite.ignore_corrupt_dbfiles = 1
+SCons.dblite.corruption_warning = corrupt_dblite_warning
+
 #XXX Get rid of the global array so this becomes re-entrant.
 sig_files = []
 
-# Handle to open database object if using the DB SConsign implementation.
-database = None
+# Info for the database SConsign implementation (now the default):
+# "DataBase" is a dictionary that maps top-level SConstruct directories
+# to open database handles.
+# "DB_Module" is the Python database module to create the handles.
+# "DB_Name" is the base name of the database file (minus any
+# extension the underlying DB module will add).
+DataBase = {}
+DB_Module = SCons.dblite
+DB_Name = ".sconsign"
+DB_sync_list = []
+
+def Get_DataBase(dir):
+    global DataBase, DB_Module, DB_Name
+    top = dir.fs.Top
+    if not os.path.isabs(DB_Name) and top.repositories:
+        mode = "c"
+        for d in [top] + top.repositories:
+            if dir.is_under(d):
+                try:
+                    return DataBase[d], mode
+                except KeyError:
+                    path = d.entry_abspath(DB_Name)
+                    try: db = DataBase[d] = DB_Module.open(path, mode)
+                    except (IOError, OSError): pass
+                    else:
+                        if mode != "r":
+                            DB_sync_list.append(db)
+                        return db, mode
+            mode = "r"
+    try:
+        return DataBase[top], "c"
+    except KeyError:
+        db = DataBase[top] = DB_Module.open(DB_Name, "c")
+        DB_sync_list.append(db)
+        return db, "c"
+    except TypeError:
+        print "DataBase =", DataBase
+        raise
+
+def Reset():
+    """Reset global state.  Used by unit tests that end up using
+    SConsign multiple times to get a clean slate for each test."""
+    global sig_files, DB_sync_list
+    sig_files = []
+    DB_sync_list = []
 
 if os.sep == '/':
     norm_entry = lambda s: s
@@ -55,9 +108,9 @@ def write():
     global sig_files
     for sig_file in sig_files:
         sig_file.write(sync=0)
-    if database:
+    for db in DB_sync_list:
         try:
-            syncmethod = database.sync
+            syncmethod = db.sync
         except AttributeError:
             pass # Not all anydbm modules have sync() methods.
         else:
@@ -94,19 +147,25 @@ class Base:
         self.entries[filename] = obj
         self.dirty = 1
 
+    def do_not_set_entry(self, filename, obj):
+        pass
+
 class DB(Base):
     """
     A Base subclass that reads and writes signature information
-    from a global .sconsign.dbm file.
+    from a global .sconsign.db* file--the actual file suffix is
+    determined by the specified database module.
     """
     def __init__(self, dir, module=None):
         Base.__init__(self, module)
 
         self.dir = dir
 
+        db, mode = Get_DataBase(dir)
+
+        tpath = norm_entry(dir.tpath)
         try:
-            global database
-            rawentries = database[norm_entry(self.dir.path)]
+            rawentries = db[tpath]
         except KeyError:
             pass
         else:
@@ -119,18 +178,25 @@ class DB(Base):
                 raise
             except:
                 SCons.Warnings.warn(SCons.Warnings.CorruptSConsignWarning,
-                                    "Ignoring corrupt sconsign entry : %s"%self.dir.path)
+                                    "Ignoring corrupt sconsign entry : %s"%self.dir.tpath)
+
+        if mode == "r":
+            # This directory is actually under a repository, which means
+            # likely they're reaching in directly for a dependency on
+            # a file there.  Don't actually set any entry info, so we
+            # won't try to write to that .sconsign.dblite file.
+            self.set_entry = self.do_not_set_entry
 
         global sig_files
         sig_files.append(self)
 
     def write(self, sync=1):
         if self.dirty:
-            global database
-            database[norm_entry(self.dir.path)] = cPickle.dumps(self.entries, 1)
+            db, mode = Get_DataBase(self.dir)
+            db[norm_entry(self.dir.tpath)] = cPickle.dumps(self.entries, 1)
             if sync:
                 try:
-                    syncmethod = database.sync
+                    syncmethod = db.sync
                 except AttributeError:
                     # Not all anydbm modules have sync() methods.
                     pass
@@ -223,19 +289,19 @@ class DirFile(Dir):
             except OSError:
                 pass
 
-ForDirectory = DirFile
+ForDirectory = DB
 
 def File(name, dbm_module=None):
     """
-    Arrange for all signatures to be stored in a global .sconsign.dbm
+    Arrange for all signatures to be stored in a global .sconsign.db*
     file.
     """
-    global database
-    if database is None:
-        if dbm_module is None:
-            import SCons.dblite
-            dbm_module = SCons.dblite
-        database = dbm_module.open(name, "c")
-
-    global ForDirectory
-    ForDirectory = DB
+    global ForDirectory, DB_Name, DB_Module
+    if name is None:
+        ForDirectory = DirFile
+        DB_Module = None
+    else:
+        ForDirectory = DB
+        DB_Name = name
+        if not dbm_module is None:
+            DB_Module = dbm_module
diff --git a/src/engine/SCons/SConsignTests.py b/src/engine/SCons/SConsignTests.py
index c2f40bd..025ad8e 100644
--- a/src/engine/SCons/SConsignTests.py
+++ b/src/engine/SCons/SConsignTests.py
@@ -28,6 +28,8 @@ import sys
 import TestCmd
 import unittest
 
+import SCons.dblite
+
 import SCons.SConsign
 
 class BuildInfo:
@@ -41,12 +43,30 @@ class DummyModule:
     def from_string(self, sig):
         return int(sig)
 
-class BaseTestCase(unittest.TestCase):
+class FS:
+    def __init__(self, top):
+        self.Top = top
+        self.Top.repositories = []
+
+class DummyNode:
+    def __init__(self, path='not_a_valid_path'):
+        self.path = path
+        self.tpath = path
+        self.fs = FS(self)
+
+class SConsignTestCase(unittest.TestCase):
+    def setUp(self):
+        self.save_cwd = os.getcwd()
+        self.test = TestCmd.TestCmd(workdir = '')
+        os.chdir(self.test.workpath(''))
+    def tearDown(self):
+        self.test.cleanup()
+        SCons.SConsign.Reset()
+        os.chdir(self.save_cwd)
+
+class BaseTestCase(SConsignTestCase):
 
     def runTest(self):
-        class DummyNode:
-            path = 'not_a_valid_path'
-
         aaa = BuildInfo('aaa')
         bbb = BuildInfo('bbb')
         bbb.arg1 = 'bbb arg1'
@@ -96,14 +116,11 @@ class BaseTestCase(unittest.TestCase):
         assert e.name == 'fff', e.name
         assert e.arg == 'fff arg', e.arg
 
-class SConsignDBTestCase(unittest.TestCase):
+class SConsignDBTestCase(SConsignTestCase):
 
     def runTest(self):
-        class DummyNode:
-            def __init__(self, path):
-                self.path = path
-        save_database = SCons.SConsign.database
-        SCons.SConsign.database = {}
+        save_DataBase = SCons.SConsign.DataBase
+        SCons.SConsign.DataBase = {}
         try:
             d1 = SCons.SConsign.DB(DummyNode('dir1'))
             d1.set_entry('aaa', BuildInfo('aaa name'))
@@ -137,14 +154,11 @@ class SConsignDBTestCase(unittest.TestCase):
             hhh = d32.get_entry('hhh')
             assert hhh.name == 'hhh name'
         finally:
-            SCons.SConsign.database = save_database
+            SCons.SConsign.DataBase = save_DataBase
 
-class SConsignDirFileTestCase(unittest.TestCase):
+class SConsignDirFileTestCase(SConsignTestCase):
 
     def runTest(self):
-        class DummyNode:
-            path = 'not_a_valid_path'
-
         foo = BuildInfo('foo')
         bar = BuildInfo('bar')
 
@@ -169,52 +183,65 @@ class SConsignDirFileTestCase(unittest.TestCase):
         assert e.arg == 'bbb arg', e.arg
 
 
-class SConsignFileTestCase(unittest.TestCase):
+class SConsignFileTestCase(SConsignTestCase):
 
     def runTest(self):
-        test = TestCmd.TestCmd(workdir = '')
+        test = self.test
         file = test.workpath('sconsign_file')
 
-        assert SCons.SConsign.database is None, SCons.SConsign.database
+        assert SCons.SConsign.DataBase == {}, SCons.SConsign.DataBase
+        assert SCons.SConsign.DB_Name == ".sconsign", SCons.SConsign.DB_Name
+        assert SCons.SConsign.DB_Module is SCons.dblite, SCons.SConsign.DB_Module
 
         SCons.SConsign.File(file)
 
-        assert not SCons.SConsign.database is SCons.dblite, SCons.SConsign.database
+        assert SCons.SConsign.DataBase == {}, SCons.SConsign.DataBase
+        assert SCons.SConsign.DB_Name is file, SCons.SConsign.DB_Name
+        assert SCons.SConsign.DB_Module is SCons.dblite, SCons.SConsign.DB_Module
+
+        SCons.SConsign.File(None)
+
+        assert SCons.SConsign.DataBase == {}, SCons.SConsign.DataBase
+        assert SCons.SConsign.DB_Name is file, SCons.SConsign.DB_Name
+        assert SCons.SConsign.DB_Module is None, SCons.SConsign.DB_Module
 
         class Fake_DBM:
             def open(self, name, mode):
                 self.name = name
                 self.mode = mode
                 return self
+            def __getitem__(self, key):
+                pass
+            def __setitem__(self, key, value):
+                pass
 
         fake_dbm = Fake_DBM()
 
         SCons.SConsign.File(file, fake_dbm)
 
-        assert not SCons.SConsign.database is None, SCons.SConsign.database
+        assert SCons.SConsign.DataBase == {}, SCons.SConsign.DataBase
+        assert SCons.SConsign.DB_Name is file, SCons.SConsign.DB_Name
+        assert SCons.SConsign.DB_Module is fake_dbm, SCons.SConsign.DB_Module
         assert not hasattr(fake_dbm, 'name'), fake_dbm
         assert not hasattr(fake_dbm, 'mode'), fake_dbm
 
-        SCons.SConsign.database = None
-
-        SCons.SConsign.File(file, fake_dbm)
+        SCons.SConsign.ForDirectory(DummyNode(test.workpath('dir')))
 
-        assert not SCons.SConsign.database is None, SCons.SConsign.database
+        assert not SCons.SConsign.DataBase is None, SCons.SConsign.DataBase
         assert fake_dbm.name == file, fake_dbm.name
         assert fake_dbm.mode == "c", fake_dbm.mode
 
 
-class writeTestCase(unittest.TestCase):
+class writeTestCase(SConsignTestCase):
 
     def runTest(self):
 
-        class DummyNode:
-            path = 'not_a_valid_path'
-
-        test = TestCmd.TestCmd(workdir = '')
+        test = self.test
         file = test.workpath('sconsign_file')
 
         class Fake_DBM:
+            def __getitem__(self, key):
+                return None
             def __setitem__(self, key, value):
                 pass
             def open(self, name, mode):
@@ -225,10 +252,10 @@ class writeTestCase(unittest.TestCase):
 
         fake_dbm = Fake_DBM()
 
-        SCons.SConsign.database = None
+        SCons.SConsign.DataBase = {}
         SCons.SConsign.File(file, fake_dbm)
 
-        f = SCons.SConsign.DirFile(DummyNode(), DummyModule())
+        f = SCons.SConsign.DB(DummyNode(), DummyModule())
 
         f.set_entry('foo', BuildInfo('foo'))
         f.set_entry('bar', BuildInfo('bar'))
diff --git a/src/engine/SCons/dblite.py b/src/engine/SCons/dblite.py
index 00f8274..637e503 100644
--- a/src/engine/SCons/dblite.py
+++ b/src/engine/SCons/dblite.py
@@ -14,6 +14,9 @@ _open = __builtin__.open # avoid name clash
 keep_all_files = 00000
 ignore_corrupt_dbfiles = 0
 
+def corruption_warning(filename):
+    print "Warning: Discarding corrupt database:", filename
+
 if hasattr(types, 'UnicodeType'):
     def is_string(s):
         t = type(s)
@@ -64,7 +67,7 @@ class dblite:
           except cPickle.UnpicklingError:
             if (ignore_corrupt_dbfiles == 0): raise
             if (ignore_corrupt_dbfiles == 1):
-              print "Warning: Discarding corrupt database:", self._file_name
+              corruption_warning(self._file_name)
 
   def __del__(self):
     if (self._needs_sync):