1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
|
"""Efficiently compare files, boolean outcome only (equal / not equal).
Tricks (used in this order):
- Files with identical type, size & mtime are assumed to be clones
- Files with different type or size cannot be identical
- We keep a cache of outcomes of earlier comparisons
- We don't fork a process to run 'cmp' but read the files ourselves
"""
import os
cache = {}
def cmp(f1, f2, shallow=1):
"""Compare two files, use the cache if possible.
Return 1 for identical files, 0 for different.
Raise exceptions if either file could not be statted, read, etc."""
s1, s2 = sig(os.stat(f1)), sig(os.stat(f2))
if s1[0] != 8 or s2[0] != 8:
# Either is a not a plain file -- always report as different
return 0
if shallow and s1 == s2:
# type, size & mtime match -- report same
return 1
if s1[:2] != s2[:2]: # Types or sizes differ, don't bother
# types or sizes differ -- report different
return 0
# same type and size -- look in the cache
key = (f1, f2)
try:
cs1, cs2, outcome = cache[key]
# cache hit
if s1 == cs1 and s2 == cs2:
# cached signatures match
return outcome
# stale cached signature(s)
except KeyError:
# cache miss
pass
# really compare
outcome = do_cmp(f1, f2)
cache[key] = s1, s2, outcome
return outcome
def sig(st):
"""Return signature (i.e., type, size, mtime) from raw stat data
0-5: st_mode, st_ino, st_dev, st_nlink, st_uid, st_gid
6-9: st_size, st_atime, st_mtime, st_ctime"""
type = st[0] / 4096
size = st[6]
mtime = st[8]
return type, size, mtime
def do_cmp(f1, f2):
"""Compare two files, really."""
bufsize = 8*1024 # Could be tuned
fp1 = open(f1, 'rb')
fp2 = open(f2, 'rb')
while 1:
b1 = fp1.read(bufsize)
b2 = fp2.read(bufsize)
if b1 != b2: return 0
if not b1: return 1
|
