Issue 1646: Block-wise signature computation for large files

1 files changed, 28 insertions, 4 deletions

diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py
index 02dcdbf..8301b15 100644
--- a/src/engine/SCons/Node/FS.py
+++ b/src/engine/SCons/Node/FS.py
@@ -2160,6 +2160,8 @@ class File(Base):
     NodeInfo = FileNodeInfo
     BuildInfo = FileBuildInfo
 
+    md5_chunksize = 64
+
     def diskcheck_match(self):
         diskcheck_match(self, self.isdir,
                         "Directory %s found where file expected.")
@@ -2233,6 +2235,23 @@ class File(Base):
             raise
         return r
 
+    def get_content_hash(self):
+        """
+        Compute and return the MD5 hash for this file.
+        """
+        if not self.rexists():
+            return SCons.Util.MD5signature('')
+        fname = self.rfile().abspath
+        try:
+            cs = SCons.Util.MD5filesignature(fname,
+                chunksize=SCons.Node.FS.File.md5_chunksize*1024)
+        except EnvironmentError, e:
+            if not e.filename:
+                e.filename = fname
+            raise
+        return cs
+        
+
     memoizer_counters.append(SCons.Memoize.CountValue('get_size'))
 
     def get_size(self):
@@ -2697,7 +2716,10 @@ class File(Base):
         if csig is None:
 
             try:
-                contents = self.get_contents()
+                if self.get_size() < SCons.Node.FS.File.md5_chunksize:
+                    contents = self.get_contents()
+                else:
+                    csig = self.get_content_hash()
             except IOError:
                 # This can happen if there's actually a directory on-disk,
                 # which can be the case if they've disabled disk checks,
@@ -2705,7 +2727,8 @@ class File(Base):
                 # create a same-named directory by mistake.
                 csig = ''
             else:
-                csig = SCons.Util.MD5signature(contents)
+                if not csig:
+                    csig = SCons.Util.MD5signature(contents)
 
         ninfo.csig = csig
 
@@ -2833,8 +2856,8 @@ class File(Base):
 
         cachedir, cachefile = self.get_build_env().get_CacheDir().cachepath(self)
         if not self.exists() and cachefile and os.path.exists(cachefile):
-            contents = open(cachefile, 'rb').read()
-            self.cachedir_csig = SCons.Util.MD5signature(contents)
+            self.cachedir_csig = SCons.Util.MD5filesignature(cachefile, \
+                SCons.Node.FS.File.md5_chunksize * 1024)
         else:
             self.cachedir_csig = self.get_csig()
         return self.cachedir_csig
@@ -2856,6 +2879,7 @@ class File(Base):
         self.cachesig = SCons.Util.MD5collect(sigs)
         return self.cachesig
 
+
 default_fs = None
 
 def get_default_fs():