Have SConsignFile() use a dblite.py module by default, so we can control the behavior. (Ralf W. Grosse-Kunstleve)

9 files changed, 284 insertions, 52 deletions

diff --git a/src/CHANGES.txt b/src/CHANGES.txt
index 0d28a0a..bba15e7 100644
--- a/src/CHANGES.txt
+++ b/src/CHANGES.txt
@@ -40,6 +40,11 @@ RELEASE 0.96 - XXX
 
   - Allow the Java() Builder to take more than one source directory.
 
+  From Ralf W. Grosse-Kunstleve:
+
+  - Have SConsignFile() use, by default, a custom "dblite.py" that we can
+    control and guarantee to work on all Python versions (or nearly so).
+
   From Bob Halley:
 
   - Make the new *FLAGS variable type work with copied Environments.
diff --git a/src/RELEASE.txt b/src/RELEASE.txt
index 09a3d6f..c7b46fd 100644
--- a/src/RELEASE.txt
+++ b/src/RELEASE.txt
@@ -27,6 +27,16 @@ RELEASE 0.96 - XXX
 
   Please note the following important changes since release 0.95:
 
+    - The SConsignFile() function now uses an internally-supplied
+      SCons.dblite module as the default DB scheme for the .sconsign file.
+      If you are using the SConsignFile() function without an explicitly
+      specified dbm_module argument, this will cause all of your targets
+      to be recompiled the first time you use SCons 0.96.  To preserve the
+      previous behavior, specify the "anydbm" module explicitly:
+
+          import anydbm
+          SConsignFile('.sconsign_file_name', anydbm)
+
     - The internal Scanner.add_skey() method longer works for the default
       scanners, which now use construction variables to hold their lists
       of suffixes.  If you had a custom Tool specification that was
diff --git a/src/engine/MANIFEST.in b/src/engine/MANIFEST.in
index a8bac0a..4942be8 100644
--- a/src/engine/MANIFEST.in
+++ b/src/engine/MANIFEST.in
@@ -2,6 +2,7 @@ SCons/__init__.py
 SCons/Action.py
 SCons/Builder.py
 SCons/Conftest.py
+SCons/dblite.py
 SCons/Debug.py
 SCons/Defaults.py
 SCons/Environment.py
diff --git a/src/engine/SCons/Environment.py b/src/engine/SCons/Environment.py
index c312f31..d214e9b 100644
--- a/src/engine/SCons/Environment.py
+++ b/src/engine/SCons/Environment.py
@@ -1153,7 +1153,7 @@ class Base:
         nkw = self.subst_kw(kw)
         return apply(SCons.Scanner.Base, nargs, nkw)
 
-    def SConsignFile(self, name=".sconsign.dbm", dbm_module=None):
+    def SConsignFile(self, name=".sconsign", dbm_module=None):
         name = self.subst(name)
         if not os.path.isabs(name):
             name = os.path.join(str(self.fs.SConstruct_dir), name)
diff --git a/src/engine/SCons/EnvironmentTests.py b/src/engine/SCons/EnvironmentTests.py
index 0995a05..d141d49 100644
--- a/src/engine/SCons/EnvironmentTests.py
+++ b/src/engine/SCons/EnvironmentTests.py
@@ -2245,6 +2245,10 @@ class EnvironmentTestCase(unittest.TestCase):
             env.SConsignFile('__$BAR', 7)
             assert fnames[4] == os.path.join(os.sep, 'dir', '__', 'File'), fnames
             assert dbms[4] == 7, dbms
+
+            env.SConsignFile()
+            assert fnames[5] == os.path.join(os.sep, 'dir', '.sconsign'), fnames
+            assert dbms[5] == None, dbms
         finally:
             SCons.Sig.SConsignFile = save_Sig_SConsignFile
 
diff --git a/src/engine/SCons/Sig/SigTests.py b/src/engine/SCons/Sig/SigTests.py
index 98465ff..76cd931 100644
--- a/src/engine/SCons/Sig/SigTests.py
+++ b/src/engine/SCons/Sig/SigTests.py
@@ -502,7 +502,7 @@ class SConsignFileTestCase(unittest.TestCase):
 
         SCons.Sig.SConsignFile(file)
 
-        assert not SCons.Sig.SConsign_db is None, SCons.Sig.SConsign_db
+        assert not SCons.Sig.SConsign_db is SCons.dblite, SCons.Sig.SConsign_db
 
         class Fake_DBM:
             def open(self, name, mode):
diff --git a/src/engine/SCons/Sig/__init__.py b/src/engine/SCons/Sig/__init__.py
index d703194..6bdfe09 100644
--- a/src/engine/SCons/Sig/__init__.py
+++ b/src/engine/SCons/Sig/__init__.py
@@ -326,8 +326,8 @@ def SConsignFile(name, dbm_module=None):
     global SConsign_db
     if SConsign_db is None:
         if dbm_module is None:
-            import anydbm
-            dbm_module = anydbm
+            import SCons.dblite
+            dbm_module = SCons.dblite
         SConsign_db = dbm_module.open(name, "c")
 
     global SConsignForDirectory
diff --git a/src/engine/SCons/dblite.py b/src/engine/SCons/dblite.py
new file mode 100644
index 0000000..df04d80
--- /dev/null
+++ b/src/engine/SCons/dblite.py
@@ -0,0 +1,149 @@
+# dblite.py module contributed by Ralf W. Grosse-Kunstleve.
+
+import cPickle
+import time
+import shutil
+import os
+import __builtin__
+
+_open = __builtin__.open # avoid name clash
+
+keep_all_files = 00000
+ignore_corrupt_dbfiles = 0
+
+class dblite:
+
+  def __init__(self, file_base_name, flag, mode):
+    assert flag in (None, "r", "w", "c", "n")
+    if (flag is None): flag = "r"
+    if file_base_name[-7:] != '.dblite':
+        file_base_name = file_base_name + '.dblite'
+    self._file_name = file_base_name
+    self._flag = flag
+    self._mode = mode
+    self._dict = {}
+    self._needs_sync = 00000
+    if (self._flag == "n"):
+      _open(self._file_name, "wb", self._mode)
+    else:
+      try:
+        f = _open(self._file_name, "rb")
+      except IOError, e:
+        if (self._flag != "c"):
+          raise e
+        _open(self._file_name, "wb", self._mode)
+      else:
+        p = f.read()
+        if (len(p) > 0):
+          try:
+            self._dict = cPickle.loads(p)
+          except:
+            if (ignore_corrupt_dbfiles == 0): raise
+            if (ignore_corrupt_dbfiles == 1):
+              print "Warning: Discarding corrupt database:", self._file_name
+
+  def __del__(self):
+    if (self._needs_sync):
+      self.sync()
+
+  def sync(self):
+    self._check_writable()
+    f = _open(self._file_name, "wb", self._mode)
+    cPickle.dump(self._dict, f, 1)
+    f.close()
+    self._needs_sync = 00000
+    if (keep_all_files):
+      shutil.copyfile(
+        self._file_name,
+        self._file_name + "_" + str(int(time.time())))
+
+  def _check_writable(self):
+    if (self._flag == "r"):
+      raise IOError("Read-only database: %s" % self._file_name)
+
+  def __getitem__(self, key):
+    return self._dict[key]
+
+  def __setitem__(self, key, value):
+    self._check_writable()
+    if (type(key) != type("")):
+      raise TypeError, "key must be a string"
+    if (type(value) != type("")):
+      raise TypeError, "value must be a string"
+    self._dict[key] = value
+    self._needs_sync = 0001
+
+  def keys(self):
+    return self._dict.keys()
+
+  def has_key(self, key):
+    return key in self._dict
+
+  def __contains__(self, key):
+    return key in self._dict
+
+  def iterkeys(self):
+    return self._dict.iterkeys()
+
+  __iter__ = iterkeys
+
+  def __len__(self):
+    return len(self._dict)
+
+def open(file, flag=None, mode=0666):
+  return dblite(file, flag, mode)
+
+def _exercise():
+  db = open("tmp", "n")
+  assert len(db) == 0
+  db["foo"] = "bar"
+  assert db["foo"] == "bar"
+  db.sync()
+  db = open("tmp", "c")
+  assert len(db) == 1
+  assert db["foo"] == "bar"
+  db["bar"] = "foo"
+  assert db["bar"] == "foo"
+  db.sync()
+  db = open("tmp", "r")
+  assert len(db) == 2
+  assert db["foo"] == "bar"
+  assert db["bar"] == "foo"
+  try:
+    db.sync()
+  except IOError, e:
+    assert str(e) == "Read-only database: tmp.dblite"
+  else:
+    raise RuntimeError, "IOError expected."
+  db = open("tmp", "w")
+  assert len(db) == 2
+  db["ping"] = "pong"
+  db.sync()
+  db = open("tmp", "r")
+  assert len(db) == 3
+  db = open("tmp", "n")
+  assert len(db) == 0
+  _open("tmp.dblite", "w")
+  db = open("tmp", "r")
+  _open("tmp.dblite", "w").write("x")
+  try:
+    db = open("tmp", "r")
+  except cPickle.UnpicklingError:
+    pass
+  else:
+    raise RuntimeError, "cPickle exception expected."
+  global ignore_corrupt_dbfiles
+  ignore_corrupt_dbfiles = 2
+  db = open("tmp", "r")
+  assert len(db) == 0
+  os.unlink("tmp.dblite")
+  try:
+    db = open("tmp", "w")
+  except IOError, e:
+    assert str(e) == "Database does not exist: tmp.dblite"
+  else:
+    raise RuntimeError, "IOError expected."
+  print "OK"
+
+if (__name__ == "__main__"):
+  _exercise()
diff --git a/src/script/sconsign.py b/src/script/sconsign.py
index 1c0a769..e3905ce 100644
--- a/src/script/sconsign.py
+++ b/src/script/sconsign.py
@@ -141,13 +141,42 @@ sys.path = libs + sys.path
 # END STANDARD SCons SCRIPT HEADER
 ##############################################################################
 
+import cPickle
+import imp
+import string
+import whichdb
+
+import SCons.Sig
+
+def my_whichdb(filename):
+    try:
+        f = open(filename + ".dblite", "rb")
+        f.close()
+        return "SCons.dblite"
+    except IOError:
+        pass
+    return _orig_whichdb(filename)
+
+_orig_whichdb = whichdb.whichdb
+whichdb.whichdb = my_whichdb
+
+def my_import(mname):
+    if '.' in mname:
+        i = string.rfind(mname, '.')
+        parent = my_import(mname[:i])
+        fp, pathname, description = imp.find_module(mname[i+1:],
+                                                    parent.__path__)
+    else:
+        fp, pathname, description = imp.find_module(mname)
+    return imp.load_module(mname, fp, pathname, description)
+
 PF_bsig      = 0x1
 PF_csig      = 0x2
 PF_timestamp = 0x4
 PF_implicit  = 0x8
 PF_all       = PF_bsig | PF_csig | PF_timestamp | PF_implicit
 
-Do_Func = None
+Do_Call = None
 Print_Directories = []
 Print_Entries = []
 Print_Flags = 0
@@ -192,38 +221,63 @@ def printentries(entries):
         for name, e in entries.items():
             printfield(name, e)
 
-import SCons.Sig
+class Do_SConsignDB:
+    def __init__(self, dbm_name, dbm):
+        self.dbm_name = dbm_name
+        self.dbm = dbm
 
-def Do_SConsignDB(name):
-    import anydbm
-    import cPickle
-    try:
-        open(name, 'rb')
-    except (IOError, OSError), e:
-        sys.stderr.write("sconsign: %s\n" % (e))
-        return
-    try:
-        db = anydbm.open(name, "r")
-    except anydbm.error, e:
-        sys.stderr.write("sconsign: ignoring invalid .sconsign.dbm file `%s': %s\n" % (name, e))
-        return
-    if Print_Directories:
-        for dir in Print_Directories:
+    def __call__(self, fname):
+        # The *dbm modules stick their own file suffixes on the names
+        # that are passed in.  This is causes us to jump through some
+        # hoops here to be able to allow the user
+        try:
+            # Try opening the specified file name.  Example:
+            #   SPECIFIED                  OPENED BY self.dbm.open()
+            #   ---------                  -------------------------
+            #   .sconsign               => .sconsign.dblite
+            #   .sconsign.dblite        => .sconsign.dblite.dblite
+            db = self.dbm.open(fname, "r")
+        except (IOError, OSError), e:
+            print_e = e
             try:
-                val = db[dir]
-            except KeyError:
-                sys.stderr.write("sconsign: no dir `%s' in `%s'\n" % (dir, args[0]))
-            else:
-                entries = cPickle.loads(val)
-                print '=== ' + dir + ':'
-                printentries(entries)
-    else:
-        keys = db.keys()
-        keys.sort()
-        for dir in keys:
-            entries = cPickle.loads(db[dir])
-            print '=== ' + dir + ':'
-            printentries(entries)
+                # That didn't work, so try opening the base name,
+                # so that if the actually passed in 'sconsign.dblite'
+                # (for example), the dbm module will put the suffix back
+                # on for us and open it anyway.
+                db = self.dbm.open(os.path.splitext(fname)[0], "r")
+            except (IOError, OSError):
+                # That didn't work either.  See if the file name
+                # they specified just exists (independent of the dbm
+                # suffix-mangling).
+                try:
+                    open(fname, "r")
+                except (IOError, OSError), e:
+                    # Nope, that file doesn't even exist, so report that
+                    # fact back.
+                    print_e = e
+                sys.stderr.write("sconsign: %s\n" % (print_e))
+                return
+        except:
+            sys.stderr.write("sconsign: ignoring invalid `%s' file `%s'\n" % (self.dbm_name, fname))
+            return
+
+        if Print_Directories:
+            for dir in Print_Directories:
+                try:
+                    val = db[dir]
+                except KeyError:
+                    sys.stderr.write("sconsign: no dir `%s' in `%s'\n" % (dir, args[0]))
+                else:
+                    self.printentries(dir, val)
+        else:
+            keys = db.keys()
+            keys.sort()
+            for dir in keys:
+                self.printentries(dir, db[dir])
+
+    def printentries(self, dir, val):
+        print '=== ' + dir + ':'
+        printentries(cPickle.loads(val))
 
 def Do_SConsignDir(name):
     try:
@@ -238,9 +292,6 @@ def Do_SConsignDir(name):
         return
     printentries(sconsign.entries)
 
-Function_Map = {'dbm'      : Do_SConsignDB,
-                'sconsign' : Do_SConsignDir}
-
 ##############################################################################
 
 import getopt
@@ -275,12 +326,19 @@ for o, a in opts:
     elif o in ('-e', '--entry'):
         Print_Entries.append(a)
     elif o in ('-f', '--format'):
-        try:
-            Do_Func = Function_Map[a]
-        except KeyError:
-            sys.stderr.write("sconsign: illegal file format `%s'\n" % a)
-            print helpstr
-            sys.exit(2)
+        Module_Map = {'dblite'   : 'SCons.dblite',
+                      'sconsign' : None}
+        dbm_name = Module_Map.get(a, a)
+        if dbm_name:
+            try:
+                dbm = my_import(dbm_name)
+            except:
+                sys.stderr.write("sconsign: illegal file format `%s'\n" % a)
+                print helpstr
+                sys.exit(2)
+            Do_Call = Do_SConsignDB(a, dbm)
+        else:
+            Do_Call = Do_SConsignDir
     elif o in ('-h', '--help'):
         print helpstr
         sys.exit(0)
@@ -295,13 +353,18 @@ for o, a in opts:
 
 if Print_Flags == 0:
     Print_Flags = PF_all
-    
-for a in args:
-    if Do_Func:
-        Do_Func(a)
-    elif a[-4:] == '.dbm':
-        Do_SConsignDB(a)
-    else:
-        Do_SConsignDir(a)
+
+if Do_Call:
+    for a in args:
+        Do_Call(a)
+else:
+    for a in args:
+        dbm_name = whichdb.whichdb(a)
+        if dbm_name:
+            Map_Module = {'SCons.dblite' : 'dblite'}
+            dbm = my_import(dbm_name)
+            Do_SConsignDB(Map_Module.get(dbm_name, dbm_name), dbm)(a)
+        else:
+            Do_SConsignDir(a)
 
 sys.exit(0)