diff options
Diffstat (limited to 'Tools/scripts/ftpmirror.py')
| -rwxr-xr-x | Tools/scripts/ftpmirror.py | 668 |
1 files changed, 334 insertions, 334 deletions
diff --git a/Tools/scripts/ftpmirror.py b/Tools/scripts/ftpmirror.py index 4b02714..c894a7b 100755 --- a/Tools/scripts/ftpmirror.py +++ b/Tools/scripts/ftpmirror.py @@ -4,7 +4,7 @@ usage: ftpmirror [-v] [-q] [-i] [-m] [-n] [-r] [-s pat] [-l username [-p passwd [-a account]]] - hostname [remotedir [localdir]] + hostname [remotedir [localdir]] -v: verbose -q: quiet -i: interactive mode @@ -28,10 +28,10 @@ from fnmatch import fnmatch # Print usage message and exit def usage(*args): - sys.stdout = sys.stderr - for msg in args: print msg - print __doc__ - sys.exit(2) + sys.stdout = sys.stderr + for msg in args: print msg + print __doc__ + sys.exit(2) verbose = 1 # 0 for -q, 2 for -v interactive = 0 @@ -42,356 +42,356 @@ skippats = ['.', '..', '.mirrorinfo'] # Main program: parse command line and start processing def main(): - global verbose, interactive, mac, rmok, nologin - try: - opts, args = getopt.getopt(sys.argv[1:], 'a:bil:mnp:qrs:v') - except getopt.error, msg: - usage(msg) - login = '' - passwd = '' - account = '' - for o, a in opts: - if o == '-l': login = a - if o == '-p': passwd = a - if o == '-a': account = a - if o == '-v': verbose = verbose + 1 - if o == '-q': verbose = 0 - if o == '-i': interactive = 1 - if o == '-m': mac = 1; nologin = 1; skippats.append('*.o') - if o == '-n': nologin = 1 - if o == '-r': rmok = 1 - if o == '-s': skippats.append(a) - if not args: usage('hostname missing') - host = args[0] - remotedir = '' - localdir = '' - if args[1:]: - remotedir = args[1] - if args[2:]: - localdir = args[2] - if args[3:]: usage('too many arguments') - # - f = ftplib.FTP() - if verbose: print 'Connecting to %s...' % `host` - f.connect(host) - if not nologin: - if verbose: - print 'Logging in as %s...' % `login or 'anonymous'` - f.login(login, passwd, account) - if verbose: print 'OK.' - pwd = f.pwd() - if verbose > 1: print 'PWD =', `pwd` - if remotedir: - if verbose > 1: print 'cwd(%s)' % `remotedir` - f.cwd(remotedir) - if verbose > 1: print 'OK.' - pwd = f.pwd() - if verbose > 1: print 'PWD =', `pwd` - # - mirrorsubdir(f, localdir) + global verbose, interactive, mac, rmok, nologin + try: + opts, args = getopt.getopt(sys.argv[1:], 'a:bil:mnp:qrs:v') + except getopt.error, msg: + usage(msg) + login = '' + passwd = '' + account = '' + for o, a in opts: + if o == '-l': login = a + if o == '-p': passwd = a + if o == '-a': account = a + if o == '-v': verbose = verbose + 1 + if o == '-q': verbose = 0 + if o == '-i': interactive = 1 + if o == '-m': mac = 1; nologin = 1; skippats.append('*.o') + if o == '-n': nologin = 1 + if o == '-r': rmok = 1 + if o == '-s': skippats.append(a) + if not args: usage('hostname missing') + host = args[0] + remotedir = '' + localdir = '' + if args[1:]: + remotedir = args[1] + if args[2:]: + localdir = args[2] + if args[3:]: usage('too many arguments') + # + f = ftplib.FTP() + if verbose: print 'Connecting to %s...' % `host` + f.connect(host) + if not nologin: + if verbose: + print 'Logging in as %s...' % `login or 'anonymous'` + f.login(login, passwd, account) + if verbose: print 'OK.' + pwd = f.pwd() + if verbose > 1: print 'PWD =', `pwd` + if remotedir: + if verbose > 1: print 'cwd(%s)' % `remotedir` + f.cwd(remotedir) + if verbose > 1: print 'OK.' + pwd = f.pwd() + if verbose > 1: print 'PWD =', `pwd` + # + mirrorsubdir(f, localdir) # Core logic: mirror one subdirectory (recursively) def mirrorsubdir(f, localdir): - pwd = f.pwd() - if localdir and not os.path.isdir(localdir): - if verbose: print 'Creating local directory', `localdir` - try: - makedir(localdir) - except os.error, msg: - print "Failed to establish local directory", `localdir` - return - infofilename = os.path.join(localdir, '.mirrorinfo') - try: - text = open(infofilename, 'r').read() - except IOError, msg: - text = '{}' - try: - info = eval(text) - except (SyntaxError, NameError): - print 'Bad mirror info in %s' % `infofilename` - info = {} - subdirs = [] - listing = [] - if verbose: print 'Listing remote directory %s...' % `pwd` - f.retrlines('LIST', listing.append) - filesfound = [] - for line in listing: - if verbose > 1: print '-->', `line` - if mac: - # Mac listing has just filenames; - # trailing / means subdirectory - filename = string.strip(line) - mode = '-' - if filename[-1:] == '/': - filename = filename[:-1] - mode = 'd' - infostuff = '' - else: - # Parse, assuming a UNIX listing - words = string.split(line, None, 8) - if len(words) < 6: - if verbose > 1: print 'Skipping short line' - continue - filename = string.lstrip(words[-1]) - i = string.find(filename, " -> ") - if i >= 0: - # words[0] had better start with 'l'... - if verbose > 1: - print 'Found symbolic link %s' % `filename` - linkto = filename[i+4:] - filename = filename[:i] - infostuff = words[-5:-1] - mode = words[0] - skip = 0 - for pat in skippats: - if fnmatch(filename, pat): - if verbose > 1: - print 'Skip pattern', `pat`, - print 'matches', `filename` - skip = 1 - break - if skip: - continue - if mode[0] == 'd': - if verbose > 1: - print 'Remembering subdirectory', `filename` - subdirs.append(filename) - continue - filesfound.append(filename) - if info.has_key(filename) and info[filename] == infostuff: - if verbose > 1: - print 'Already have this version of',`filename` - continue - fullname = os.path.join(localdir, filename) - tempname = os.path.join(localdir, '@'+filename) - if interactive: - doit = askabout('file', filename, pwd) - if not doit: - if not info.has_key(filename): - info[filename] = 'Not retrieved' - continue - try: - os.unlink(tempname) - except os.error: - pass - if mode[0] == 'l': - if verbose: - print "Creating symlink %s -> %s" % ( - `filename`, `linkto`) - try: - os.symlink(linkto, tempname) - except IOError, msg: - print "Can't create %s: %s" % ( - `tempname`, str(msg)) - continue - else: - try: - fp = open(tempname, 'wb') - except IOError, msg: - print "Can't create %s: %s" % ( - `tempname`, str(msg)) - continue - if verbose: - print 'Retrieving %s from %s as %s...' % \ - (`filename`, `pwd`, `fullname`) - if verbose: - fp1 = LoggingFile(fp, 1024, sys.stdout) - else: - fp1 = fp - t0 = time.time() - try: - f.retrbinary('RETR ' + filename, - fp1.write, 8*1024) - except ftplib.error_perm, msg: - print msg - t1 = time.time() - bytes = fp.tell() - fp.close() - if fp1 != fp: - fp1.close() - try: - os.unlink(fullname) - except os.error: - pass # Ignore the error - try: - os.rename(tempname, fullname) - except os.error, msg: - print "Can't rename %s to %s: %s" % (`tempname`, - `fullname`, - str(msg)) - continue - info[filename] = infostuff - writedict(info, infofilename) - if verbose and mode[0] != 'l': - dt = t1 - t0 - kbytes = bytes / 1024.0 - print int(round(kbytes)), - print 'Kbytes in', - print int(round(dt)), - print 'seconds', - if t1 > t0: - print '(~%d Kbytes/sec)' % \ - int(round(kbytes/dt),) - print - # - # Remove files from info that are no longer remote - deletions = 0 - for filename in info.keys(): - if filename not in filesfound: - if verbose: - print "Removing obsolete info entry for", - print `filename`, "in", `localdir or "."` - del info[filename] - deletions = deletions + 1 - if deletions: - writedict(info, infofilename) - # - # Remove local files that are no longer in the remote directory - try: - if not localdir: names = os.listdir(os.curdir) - else: names = os.listdir(localdir) - except os.error: - names = [] - for name in names: - if name[0] == '.' or info.has_key(name) or name in subdirs: - continue - skip = 0 - for pat in skippats: - if fnmatch(name, pat): - if verbose > 1: - print 'Skip pattern', `pat`, - print 'matches', `name` - skip = 1 - break - if skip: - continue - fullname = os.path.join(localdir, name) - if not rmok: - if verbose: - print 'Local file', `fullname`, - print 'is no longer pertinent' - continue - if verbose: print 'Removing local file/dir', `fullname` - remove(fullname) - # - # Recursively mirror subdirectories - for subdir in subdirs: - if interactive: - doit = askabout('subdirectory', subdir, pwd) - if not doit: continue - if verbose: print 'Processing subdirectory', `subdir` - localsubdir = os.path.join(localdir, subdir) - pwd = f.pwd() - if verbose > 1: - print 'Remote directory now:', `pwd` - print 'Remote cwd', `subdir` - try: - f.cwd(subdir) - except ftplib.error_perm, msg: - print "Can't chdir to", `subdir`, ":", `msg` - else: - if verbose: print 'Mirroring as', `localsubdir` - mirrorsubdir(f, localsubdir) - if verbose > 1: print 'Remote cwd ..' - f.cwd('..') - newpwd = f.pwd() - if newpwd != pwd: - print 'Ended up in wrong directory after cd + cd ..' - print 'Giving up now.' - break - else: - if verbose > 1: print 'OK.' + pwd = f.pwd() + if localdir and not os.path.isdir(localdir): + if verbose: print 'Creating local directory', `localdir` + try: + makedir(localdir) + except os.error, msg: + print "Failed to establish local directory", `localdir` + return + infofilename = os.path.join(localdir, '.mirrorinfo') + try: + text = open(infofilename, 'r').read() + except IOError, msg: + text = '{}' + try: + info = eval(text) + except (SyntaxError, NameError): + print 'Bad mirror info in %s' % `infofilename` + info = {} + subdirs = [] + listing = [] + if verbose: print 'Listing remote directory %s...' % `pwd` + f.retrlines('LIST', listing.append) + filesfound = [] + for line in listing: + if verbose > 1: print '-->', `line` + if mac: + # Mac listing has just filenames; + # trailing / means subdirectory + filename = string.strip(line) + mode = '-' + if filename[-1:] == '/': + filename = filename[:-1] + mode = 'd' + infostuff = '' + else: + # Parse, assuming a UNIX listing + words = string.split(line, None, 8) + if len(words) < 6: + if verbose > 1: print 'Skipping short line' + continue + filename = string.lstrip(words[-1]) + i = string.find(filename, " -> ") + if i >= 0: + # words[0] had better start with 'l'... + if verbose > 1: + print 'Found symbolic link %s' % `filename` + linkto = filename[i+4:] + filename = filename[:i] + infostuff = words[-5:-1] + mode = words[0] + skip = 0 + for pat in skippats: + if fnmatch(filename, pat): + if verbose > 1: + print 'Skip pattern', `pat`, + print 'matches', `filename` + skip = 1 + break + if skip: + continue + if mode[0] == 'd': + if verbose > 1: + print 'Remembering subdirectory', `filename` + subdirs.append(filename) + continue + filesfound.append(filename) + if info.has_key(filename) and info[filename] == infostuff: + if verbose > 1: + print 'Already have this version of',`filename` + continue + fullname = os.path.join(localdir, filename) + tempname = os.path.join(localdir, '@'+filename) + if interactive: + doit = askabout('file', filename, pwd) + if not doit: + if not info.has_key(filename): + info[filename] = 'Not retrieved' + continue + try: + os.unlink(tempname) + except os.error: + pass + if mode[0] == 'l': + if verbose: + print "Creating symlink %s -> %s" % ( + `filename`, `linkto`) + try: + os.symlink(linkto, tempname) + except IOError, msg: + print "Can't create %s: %s" % ( + `tempname`, str(msg)) + continue + else: + try: + fp = open(tempname, 'wb') + except IOError, msg: + print "Can't create %s: %s" % ( + `tempname`, str(msg)) + continue + if verbose: + print 'Retrieving %s from %s as %s...' % \ + (`filename`, `pwd`, `fullname`) + if verbose: + fp1 = LoggingFile(fp, 1024, sys.stdout) + else: + fp1 = fp + t0 = time.time() + try: + f.retrbinary('RETR ' + filename, + fp1.write, 8*1024) + except ftplib.error_perm, msg: + print msg + t1 = time.time() + bytes = fp.tell() + fp.close() + if fp1 != fp: + fp1.close() + try: + os.unlink(fullname) + except os.error: + pass # Ignore the error + try: + os.rename(tempname, fullname) + except os.error, msg: + print "Can't rename %s to %s: %s" % (`tempname`, + `fullname`, + str(msg)) + continue + info[filename] = infostuff + writedict(info, infofilename) + if verbose and mode[0] != 'l': + dt = t1 - t0 + kbytes = bytes / 1024.0 + print int(round(kbytes)), + print 'Kbytes in', + print int(round(dt)), + print 'seconds', + if t1 > t0: + print '(~%d Kbytes/sec)' % \ + int(round(kbytes/dt),) + print + # + # Remove files from info that are no longer remote + deletions = 0 + for filename in info.keys(): + if filename not in filesfound: + if verbose: + print "Removing obsolete info entry for", + print `filename`, "in", `localdir or "."` + del info[filename] + deletions = deletions + 1 + if deletions: + writedict(info, infofilename) + # + # Remove local files that are no longer in the remote directory + try: + if not localdir: names = os.listdir(os.curdir) + else: names = os.listdir(localdir) + except os.error: + names = [] + for name in names: + if name[0] == '.' or info.has_key(name) or name in subdirs: + continue + skip = 0 + for pat in skippats: + if fnmatch(name, pat): + if verbose > 1: + print 'Skip pattern', `pat`, + print 'matches', `name` + skip = 1 + break + if skip: + continue + fullname = os.path.join(localdir, name) + if not rmok: + if verbose: + print 'Local file', `fullname`, + print 'is no longer pertinent' + continue + if verbose: print 'Removing local file/dir', `fullname` + remove(fullname) + # + # Recursively mirror subdirectories + for subdir in subdirs: + if interactive: + doit = askabout('subdirectory', subdir, pwd) + if not doit: continue + if verbose: print 'Processing subdirectory', `subdir` + localsubdir = os.path.join(localdir, subdir) + pwd = f.pwd() + if verbose > 1: + print 'Remote directory now:', `pwd` + print 'Remote cwd', `subdir` + try: + f.cwd(subdir) + except ftplib.error_perm, msg: + print "Can't chdir to", `subdir`, ":", `msg` + else: + if verbose: print 'Mirroring as', `localsubdir` + mirrorsubdir(f, localsubdir) + if verbose > 1: print 'Remote cwd ..' + f.cwd('..') + newpwd = f.pwd() + if newpwd != pwd: + print 'Ended up in wrong directory after cd + cd ..' + print 'Giving up now.' + break + else: + if verbose > 1: print 'OK.' # Helper to remove a file or directory tree def remove(fullname): - if os.path.isdir(fullname) and not os.path.islink(fullname): - try: - names = os.listdir(fullname) - except os.error: - names = [] - ok = 1 - for name in names: - if not remove(os.path.join(fullname, name)): - ok = 0 - if not ok: - return 0 - try: - os.rmdir(fullname) - except os.error, msg: - print "Can't remove local directory %s: %s" % \ - (`fullname`, str(msg)) - return 0 - else: - try: - os.unlink(fullname) - except os.error, msg: - print "Can't remove local file %s: %s" % \ - (`fullname`, str(msg)) - return 0 - return 1 + if os.path.isdir(fullname) and not os.path.islink(fullname): + try: + names = os.listdir(fullname) + except os.error: + names = [] + ok = 1 + for name in names: + if not remove(os.path.join(fullname, name)): + ok = 0 + if not ok: + return 0 + try: + os.rmdir(fullname) + except os.error, msg: + print "Can't remove local directory %s: %s" % \ + (`fullname`, str(msg)) + return 0 + else: + try: + os.unlink(fullname) + except os.error, msg: + print "Can't remove local file %s: %s" % \ + (`fullname`, str(msg)) + return 0 + return 1 # Wrapper around a file for writing to write a hash sign every block. class LoggingFile: - def __init__(self, fp, blocksize, outfp): - self.fp = fp - self.bytes = 0 - self.hashes = 0 - self.blocksize = blocksize - self.outfp = outfp - def write(self, data): - self.bytes = self.bytes + len(data) - hashes = int(self.bytes) / self.blocksize - while hashes > self.hashes: - self.outfp.write('#') - self.outfp.flush() - self.hashes = self.hashes + 1 - self.fp.write(data) - def close(self): - self.outfp.write('\n') + def __init__(self, fp, blocksize, outfp): + self.fp = fp + self.bytes = 0 + self.hashes = 0 + self.blocksize = blocksize + self.outfp = outfp + def write(self, data): + self.bytes = self.bytes + len(data) + hashes = int(self.bytes) / self.blocksize + while hashes > self.hashes: + self.outfp.write('#') + self.outfp.flush() + self.hashes = self.hashes + 1 + self.fp.write(data) + def close(self): + self.outfp.write('\n') # Ask permission to download a file. def askabout(filetype, filename, pwd): - prompt = 'Retrieve %s %s from %s ? [ny] ' % (filetype, filename, pwd) - while 1: - reply = string.lower(string.strip(raw_input(prompt))) - if reply in ['y', 'ye', 'yes']: - return 1 - if reply in ['', 'n', 'no', 'nop', 'nope']: - return 0 - print 'Please answer yes or no.' + prompt = 'Retrieve %s %s from %s ? [ny] ' % (filetype, filename, pwd) + while 1: + reply = string.lower(string.strip(raw_input(prompt))) + if reply in ['y', 'ye', 'yes']: + return 1 + if reply in ['', 'n', 'no', 'nop', 'nope']: + return 0 + print 'Please answer yes or no.' # Create a directory if it doesn't exist. Recursively create the # parent directory as well if needed. def makedir(pathname): - if os.path.isdir(pathname): - return - dirname = os.path.dirname(pathname) - if dirname: makedir(dirname) - os.mkdir(pathname, 0777) + if os.path.isdir(pathname): + return + dirname = os.path.dirname(pathname) + if dirname: makedir(dirname) + os.mkdir(pathname, 0777) # Write a dictionary to a file in a way that can be read back using # rval() but is still somewhat readable (i.e. not a single long line). # Also creates a backup file. def writedict(dict, filename): - dir, file = os.path.split(filename) - tempname = os.path.join(dir, '@' + file) - backup = os.path.join(dir, file + '~') - try: - os.unlink(backup) - except os.error: - pass - fp = open(tempname, 'w') - fp.write('{\n') - for key, value in dict.items(): - fp.write('%s: %s,\n' % (`key`, `value`)) - fp.write('}\n') - fp.close() - try: - os.rename(filename, backup) - except os.error: - pass - os.rename(tempname, filename) + dir, file = os.path.split(filename) + tempname = os.path.join(dir, '@' + file) + backup = os.path.join(dir, file + '~') + try: + os.unlink(backup) + except os.error: + pass + fp = open(tempname, 'w') + fp.write('{\n') + for key, value in dict.items(): + fp.write('%s: %s,\n' % (`key`, `value`)) + fp.write('}\n') + fp.close() + try: + os.rename(filename, backup) + except os.error: + pass + os.rename(tempname, filename) if __name__ == '__main__': - main() + main() |