diff options
Diffstat (limited to 'Tools')
| -rw-r--r-- | Tools/scripts/README | 1 | ||||
| -rwxr-xr-x | Tools/scripts/ftpmirror.py | 299 |
2 files changed, 300 insertions, 0 deletions
diff --git a/Tools/scripts/README b/Tools/scripts/README index 58059ee..005edf5 100644 --- a/Tools/scripts/README +++ b/Tools/scripts/README @@ -14,6 +14,7 @@ findlinksto.py Recursively find symbolic links to a given path prefix fixcid.py Massive identifier substitution on C source files fixheader.py Add some cpp magic to a C include file fixps.py Fix Python scripts' first line (if #!) +ftpmirror.py FTP mirror script h2py.py Translate #define's into Python assignments ifdef.py Remove #if(n)def groups from C sources linktree.py Make a copy of a tree with links to original files diff --git a/Tools/scripts/ftpmirror.py b/Tools/scripts/ftpmirror.py new file mode 100755 index 0000000..6741478 --- /dev/null +++ b/Tools/scripts/ftpmirror.py @@ -0,0 +1,299 @@ +#! /usr/local/bin/python + +# Mirror a remote ftp subtree into a local directory tree. +# Basic usage: ftpmirror [options] host remotedir localdir +# +# XXX To do: +# - handle symbolic links +# - back up .mirrorinfo before overwriting +# - use pickles for .mirrorinfo? + +import os +import sys +import time +import getopt +import string +import ftplib +from fnmatch import fnmatch + +usage_msg = """ +usage: ftpmirror [-v] [-q] [-i] [-m] [-n] [-r] [-s pat] + [-l username [-p passwd [-a account]]] + hostname [remotedir [localdir]] +-v: verbose +-q: quiet +-i: interactive mode +-m: macintosh server (NCSA telnet 2.4) (implies -n -s '*.o') +-n: don't log in +-r: remove files no longer pertinent +-l username [-p passwd [-a account]]: login info (default anonymous ftp) +-s pat: skip files matching pattern +hostname: remote host +remotedir: remote directory (default initial) +localdir: local directory (default current) +""" +def usage(*args): + sys.stdout = sys.stderr + for msg in args: print msg + print usage_msg + sys.exit(2) + +verbose = 1 # 0 for -q, 2 for -v +interactive = 0 +mac = 0 +rmok = 0 +nologin = 0 +skippats = [] + +def main(): + global verbose, interactive, mac, rmok, nologin + try: + opts, args = getopt.getopt(sys.argv[1:], 'a:bil:mnp:qrs:v') + except getopt.error, msg: + usage(msg) + login = '' + passwd = '' + account = '' + for o, a in opts: + if o == '-l': login = a + if o == '-p': passwd = a + if o == '-a': account = a + if o == '-v': verbose = verbose + 1 + if o == '-q': verbose = 0 + if o == '-i': interactive = 1 + if o == '-m': mac = 1; nologin = 1; skippats.append('*.o') + if o == '-n': nologin = 1 + if o == '-r': rmok = 1 + if o == '-s': skippats.append(a) + if not args: usage('hostname missing') + host = args[0] + remotedir = '' + localdir = '' + if args[1:]: + remotedir = args[1] + if args[2:]: + localdir = args[2] + if args[3:]: usage('too many arguments') + # + f = ftplib.FTP() + if verbose: print 'Connecting to %s...' % host + f.connect(host) + if not nologin: + if verbose: + print 'Logging in as %s...' % (login or 'anonymous') + f.login(login, passwd, account) + if verbose: print 'OK.' + pwd = f.pwd() + if verbose > 1: print 'PWD =', `pwd` + if remotedir: + if verbose > 1: print 'cwd(%s)' % `remotedir` + f.cwd(remotedir) + if verbose > 1: print 'OK.' + pwd = f.pwd() + if verbose > 1: print 'PWD =', `pwd` + # + mirrorsubdir(f, localdir) + +def mirrorsubdir(f, localdir): + pwd = f.pwd() + if localdir and not os.path.isdir(localdir): + if verbose: print 'Creating local directory', localdir + makedir(localdir) + infofilename = os.path.join(localdir, '.mirrorinfo') + try: + text = open(infofilename, 'r').read() + except IOError, msg: + text = '{}' + try: + info = eval(text) + except (SyntaxError, NameError): + print 'Bad mirror info in %s' % infofilename + info = {} + subdirs = [] + listing = [] + if verbose: print 'Listing remote directory %s...' % pwd + f.retrlines('LIST', listing.append) + for line in listing: + if verbose > 1: print '-->', `line` + if mac: + # Mac listing has just filenames; + # trailing / means subdirectory + filename = string.strip(line) + mode = '-' + if filename[-1:] == '/': + filename = filename[:-1] + mode = 'd' + infostuff = '' + else: + # Parse, assuming a UNIX listing + words = string.split(line) + if len(words) < 6: + if verbose > 1: print 'Skipping short line' + continue + if words[-2] == '->': + if verbose > 1: + print 'Skipping symbolic link %s -> %s' % \ + (words[-3], words[-1]) + continue + filename = words[-1] + if filename in ('.', '..'): + if verbose > 1: print 'Skipping . or ..' + continue + infostuff = words[-5:-1] + mode = words[0] + skip = 0 + for pat in skippats: + if fnmatch(filename, pat): + if verbose > 1: + print 'Skip pattern', pat, + print 'matches', filename + skip = 1 + break + if skip: + continue + if mode[0] == 'd': + if verbose > 1: + print 'Remembering subdirectory', filename + subdirs.append(filename) + continue + if info.has_key(filename) and info[filename] == infostuff: + if verbose > 1: + print 'Already have this version of', filename + continue + fullname = os.path.join(localdir, filename) + if interactive: + doit = askabout('file', filename, pwd) + if not doit: + if not info.has_key(filename): + info[filename] = 'Not retrieved' + continue + try: + fp = open(fullname, 'w') + except IOError, msg: + print "Can't create %s: %s" % (fullname, str(msg)) + continue + if verbose: + print 'Retrieving %s from %s as %s...' % \ + (filename, pwd, fullname) + if verbose: + fp1 = LoggingFile(fp, 1024, sys.stdout) + else: + fp1 = fp + t0 = time.time() + f.retrbinary('RETR ' + filename, fp1.write, 8*1024) + t1 = time.time() + bytes = fp.tell() + fp.close() + if fp1 != fp: + fp1.close() + info[filename] = infostuff + writedict(info, infofilename) + if verbose: + dt = t1 - t0 + kbytes = bytes / 1024.0 + print int(round(kbytes)), + print 'Kbytes in', + print int(round(dt)), + print 'seconds', + if t1 > t0: + print '(~%d Kbytes/sec)' % \ + int(round(kbytes/dt),) + print + # + # Remove local files that are no longer in the remote directory + if not localdir: names = os.listdir(os.curdir) + else: names = os.listdir(localdir) + for name in names: + if name[0] == '.' or info.has_key(name) or name in subdirs: + continue + fullname = os.path.join(localdir, name) + if not rmok: + if verbose: + print 'Local file', fullname, + print 'is no longer pertinent' + continue + if verbose: print 'Removing local file', fullname + try: + os.unlink(fullname) + except os.error, msg: + print "Can't remove local file %s: %s" % \ + (fullname, str(msg)) + # + # Recursively mirror subdirectories + for subdir in subdirs: + if interactive: + doit = askabout('subdirectory', subdir, pwd) + if not doit: continue + if verbose: print 'Processing subdirectory', subdir + localsubdir = os.path.join(localdir, subdir) + pwd = f.pwd() + if verbose > 1: + print 'Remote directory now:', pwd + print 'Remote cwd', subdir + try: + f.cwd(subdir) + except ftplib.error_perm, msg: + print "Can't chdir to", subdir, ":", msg + else: + if verbose: print 'Mirroring as', localsubdir + mirrorsubdir(f, localsubdir) + if verbose > 1: print 'Remote cwd ..' + f.cwd('..') + newpwd = f.pwd() + if newpwd != pwd: + print 'Ended up in wrong directory after cd + cd ..' + print 'Giving up now.' + break + else: + if verbose > 1: print 'OK.' + +# Wrapper around a file for writing to write a hash sign every block. +class LoggingFile: + def __init__(self, fp, blocksize, outfp): + self.fp = fp + self.bytes = 0 + self.hashes = 0 + self.blocksize = blocksize + self.outfp = outfp + def write(self, data): + self.bytes = self.bytes + len(data) + hashes = int(self.bytes) / self.blocksize + while hashes > self.hashes: + self.outfp.write('#') + self.outfp.flush() + self.hashes = self.hashes + 1 + self.fp.write(data) + def close(self): + self.outfp.write('\n') + +# Ask permission to download a file. +def askabout(filetype, filename, pwd): + prompt = 'Retrieve %s %s from %s ? [ny] ' % (filetype, filename, pwd) + while 1: + reply = string.lower(string.strip(raw_input(prompt))) + if reply in ['y', 'ye', 'yes']: + return 1 + if reply in ['', 'n', 'no', 'nop', 'nope']: + return 0 + print 'Please answer yes or no.' + +# Create a directory if it doesn't exist. Recursively create the +# parent directory as well if needed. +def makedir(pathname): + if os.path.isdir(pathname): + return + dirname = os.path.dirname(pathname) + if dirname: makedir(dirname) + os.mkdir(pathname, 0777) + +# Write a dictionary to a file in a way that can be read back using +# rval() but is still somewhat readable (i.e. not a single long line). +def writedict(dict, filename): + fp = open(filename, 'w') + fp.write('{\n') + for key, value in dict.items(): + fp.write('%s: %s,\n' % (`key`, `value`)) + fp.write('}\n') + fp.close() + +main() |