diff options
Diffstat (limited to 'Tools/webchecker/webchecker.py')
| -rwxr-xr-x | Tools/webchecker/webchecker.py | 758 |
1 files changed, 379 insertions, 379 deletions
diff --git a/Tools/webchecker/webchecker.py b/Tools/webchecker/webchecker.py index 23dcf80..5459e97 100755 --- a/Tools/webchecker/webchecker.py +++ b/Tools/webchecker/webchecker.py @@ -116,17 +116,17 @@ import robotparser if __version__[0] == '$': _v = string.split(__version__) if len(_v) == 3: - __version__ = _v[1] + __version__ = _v[1] # Tunable parameters -DEFROOT = "file:/usr/local/etc/httpd/htdocs/" # Default root URL -CHECKEXT = 1 # Check external references (1 deep) -VERBOSE = 1 # Verbosity level (0-3) -MAXPAGE = 150000 # Ignore files bigger than this -ROUNDSIZE = 50 # Number of links processed per round -DUMPFILE = "@webchecker.pickle" # Pickled checkpoint -AGENTNAME = "webchecker" # Agent name for robots.txt parser +DEFROOT = "file:/usr/local/etc/httpd/htdocs/" # Default root URL +CHECKEXT = 1 # Check external references (1 deep) +VERBOSE = 1 # Verbosity level (0-3) +MAXPAGE = 150000 # Ignore files bigger than this +ROUNDSIZE = 50 # Number of links processed per round +DUMPFILE = "@webchecker.pickle" # Pickled checkpoint +AGENTNAME = "webchecker" # Agent name for robots.txt parser # Global variables @@ -142,76 +142,76 @@ def main(): norun = 0 try: - opts, args = getopt.getopt(sys.argv[1:], 'Rd:m:nqr:vx') + opts, args = getopt.getopt(sys.argv[1:], 'Rd:m:nqr:vx') except getopt.error, msg: - sys.stdout = sys.stderr - print msg - print __doc__%globals() - sys.exit(2) + sys.stdout = sys.stderr + print msg + print __doc__%globals() + sys.exit(2) for o, a in opts: - if o == '-R': - restart = 1 - if o == '-d': - dumpfile = a - if o == '-m': - maxpage = string.atoi(a) - if o == '-n': - norun = 1 - if o == '-q': - verbose = 0 - if o == '-r': - roundsize = string.atoi(a) - if o == '-v': - verbose = verbose + 1 - if o == '-x': - checkext = not checkext + if o == '-R': + restart = 1 + if o == '-d': + dumpfile = a + if o == '-m': + maxpage = string.atoi(a) + if o == '-n': + norun = 1 + if o == '-q': + verbose = 0 + if o == '-r': + roundsize = string.atoi(a) + if o == '-v': + verbose = verbose + 1 + if o == '-x': + checkext = not checkext if verbose > 0: - print AGENTNAME, "version", __version__ + print AGENTNAME, "version", __version__ if restart: - c = load_pickle(dumpfile=dumpfile, verbose=verbose) + c = load_pickle(dumpfile=dumpfile, verbose=verbose) else: - c = Checker() + c = Checker() c.setflags(checkext=checkext, verbose=verbose, - maxpage=maxpage, roundsize=roundsize) + maxpage=maxpage, roundsize=roundsize) if not restart and not args: - args.append(DEFROOT) + args.append(DEFROOT) for arg in args: - c.addroot(arg) + c.addroot(arg) if not norun: - try: - c.run() - except KeyboardInterrupt: - if verbose > 0: - print "[run interrupted]" + try: + c.run() + except KeyboardInterrupt: + if verbose > 0: + print "[run interrupted]" try: - c.report() + c.report() except KeyboardInterrupt: - if verbose > 0: - print "[report interrupted]" + if verbose > 0: + print "[report interrupted]" if c.save_pickle(dumpfile): - if dumpfile == DUMPFILE: - print "Use ``%s -R'' to restart." % sys.argv[0] - else: - print "Use ``%s -R -d %s'' to restart." % (sys.argv[0], dumpfile) + if dumpfile == DUMPFILE: + print "Use ``%s -R'' to restart." % sys.argv[0] + else: + print "Use ``%s -R -d %s'' to restart." % (sys.argv[0], dumpfile) def load_pickle(dumpfile=DUMPFILE, verbose=VERBOSE): if verbose > 0: - print "Loading checkpoint from %s ..." % dumpfile + print "Loading checkpoint from %s ..." % dumpfile f = open(dumpfile, "rb") c = pickle.load(f) f.close() if verbose > 0: - print "Done." - print "Root:", string.join(c.roots, "\n ") + print "Done." + print "Root:", string.join(c.roots, "\n ") return c @@ -225,364 +225,364 @@ class Checker: validflags = tuple(dir()) def __init__(self): - self.reset() + self.reset() def setflags(self, **kw): - for key in kw.keys(): - if key not in self.validflags: - raise NameError, "invalid keyword argument: %s" % str(key) - for key, value in kw.items(): - setattr(self, key, value) + for key in kw.keys(): + if key not in self.validflags: + raise NameError, "invalid keyword argument: %s" % str(key) + for key, value in kw.items(): + setattr(self, key, value) def reset(self): - self.roots = [] - self.todo = {} - self.done = {} - self.bad = {} - self.round = 0 - # The following are not pickled: - self.robots = {} - self.errors = {} - self.urlopener = MyURLopener() - self.changed = 0 + self.roots = [] + self.todo = {} + self.done = {} + self.bad = {} + self.round = 0 + # The following are not pickled: + self.robots = {} + self.errors = {} + self.urlopener = MyURLopener() + self.changed = 0 def __getstate__(self): - return (self.roots, self.todo, self.done, self.bad, self.round) + return (self.roots, self.todo, self.done, self.bad, self.round) def __setstate__(self, state): - self.reset() - (self.roots, self.todo, self.done, self.bad, self.round) = state - for root in self.roots: - self.addrobot(root) - for url in self.bad.keys(): - self.markerror(url) + self.reset() + (self.roots, self.todo, self.done, self.bad, self.round) = state + for root in self.roots: + self.addrobot(root) + for url in self.bad.keys(): + self.markerror(url) def addroot(self, root): - if root not in self.roots: - troot = root - scheme, netloc, path, params, query, fragment = \ - urlparse.urlparse(root) - i = string.rfind(path, "/") + 1 - if 0 < i < len(path): - path = path[:i] - troot = urlparse.urlunparse((scheme, netloc, path, - params, query, fragment)) - self.roots.append(troot) - self.addrobot(root) - self.newlink(root, ("<root>", root)) + if root not in self.roots: + troot = root + scheme, netloc, path, params, query, fragment = \ + urlparse.urlparse(root) + i = string.rfind(path, "/") + 1 + if 0 < i < len(path): + path = path[:i] + troot = urlparse.urlunparse((scheme, netloc, path, + params, query, fragment)) + self.roots.append(troot) + self.addrobot(root) + self.newlink(root, ("<root>", root)) def addrobot(self, root): - root = urlparse.urljoin(root, "/") - if self.robots.has_key(root): return - url = urlparse.urljoin(root, "/robots.txt") - self.robots[root] = rp = robotparser.RobotFileParser() - if self.verbose > 2: - print "Parsing", url - rp.debug = self.verbose > 3 - rp.set_url(url) - try: - rp.read() - except IOError, msg: - if self.verbose > 1: - print "I/O error parsing", url, ":", msg + root = urlparse.urljoin(root, "/") + if self.robots.has_key(root): return + url = urlparse.urljoin(root, "/robots.txt") + self.robots[root] = rp = robotparser.RobotFileParser() + if self.verbose > 2: + print "Parsing", url + rp.debug = self.verbose > 3 + rp.set_url(url) + try: + rp.read() + except IOError, msg: + if self.verbose > 1: + print "I/O error parsing", url, ":", msg def run(self): - while self.todo: - self.round = self.round + 1 - if self.verbose > 0: - print - print "Round %d (%s)" % (self.round, self.status()) - print - urls = self.todo.keys()[:self.roundsize] - for url in urls: - self.dopage(url) + while self.todo: + self.round = self.round + 1 + if self.verbose > 0: + print + print "Round %d (%s)" % (self.round, self.status()) + print + urls = self.todo.keys()[:self.roundsize] + for url in urls: + self.dopage(url) def status(self): - return "%d total, %d to do, %d done, %d bad" % ( - len(self.todo)+len(self.done), - len(self.todo), len(self.done), - len(self.bad)) + return "%d total, %d to do, %d done, %d bad" % ( + len(self.todo)+len(self.done), + len(self.todo), len(self.done), + len(self.bad)) def report(self): - print - if not self.todo: print "Final", - else: print "Interim", - print "Report (%s)" % self.status() - self.report_errors() + print + if not self.todo: print "Final", + else: print "Interim", + print "Report (%s)" % self.status() + self.report_errors() def report_errors(self): - if not self.bad: - print - print "No errors" - return - print - print "Error Report:" - sources = self.errors.keys() - sources.sort() - for source in sources: - triples = self.errors[source] - print - if len(triples) > 1: - print len(triples), "Errors in", source - else: - print "Error in", source - for url, rawlink, msg in triples: - print " HREF", url, - if rawlink != url: print "(%s)" % rawlink, - print - print " msg", msg + if not self.bad: + print + print "No errors" + return + print + print "Error Report:" + sources = self.errors.keys() + sources.sort() + for source in sources: + triples = self.errors[source] + print + if len(triples) > 1: + print len(triples), "Errors in", source + else: + print "Error in", source + for url, rawlink, msg in triples: + print " HREF", url, + if rawlink != url: print "(%s)" % rawlink, + print + print " msg", msg def dopage(self, url): - if self.verbose > 1: - if self.verbose > 2: - self.show("Check ", url, " from", self.todo[url]) - else: - print "Check ", url - page = self.getpage(url) - if page: - for info in page.getlinkinfos(): - link, rawlink = info - origin = url, rawlink - self.newlink(link, origin) - self.markdone(url) + if self.verbose > 1: + if self.verbose > 2: + self.show("Check ", url, " from", self.todo[url]) + else: + print "Check ", url + page = self.getpage(url) + if page: + for info in page.getlinkinfos(): + link, rawlink = info + origin = url, rawlink + self.newlink(link, origin) + self.markdone(url) def newlink(self, url, origin): - if self.done.has_key(url): - self.newdonelink(url, origin) - else: - self.newtodolink(url, origin) + if self.done.has_key(url): + self.newdonelink(url, origin) + else: + self.newtodolink(url, origin) def newdonelink(self, url, origin): - self.done[url].append(origin) - if self.verbose > 3: - print " Done link", url + self.done[url].append(origin) + if self.verbose > 3: + print " Done link", url def newtodolink(self, url, origin): - if self.todo.has_key(url): - self.todo[url].append(origin) - if self.verbose > 3: - print " Seen todo link", url - else: - self.todo[url] = [origin] - if self.verbose > 3: - print " New todo link", url + if self.todo.has_key(url): + self.todo[url].append(origin) + if self.verbose > 3: + print " Seen todo link", url + else: + self.todo[url] = [origin] + if self.verbose > 3: + print " New todo link", url def markdone(self, url): - self.done[url] = self.todo[url] - del self.todo[url] - self.changed = 1 + self.done[url] = self.todo[url] + del self.todo[url] + self.changed = 1 def inroots(self, url): - for root in self.roots: - if url[:len(root)] == root: - root = urlparse.urljoin(root, "/") - return self.robots[root].can_fetch(AGENTNAME, url) - return 0 + for root in self.roots: + if url[:len(root)] == root: + root = urlparse.urljoin(root, "/") + return self.robots[root].can_fetch(AGENTNAME, url) + return 0 def getpage(self, url): - if url[:7] == 'mailto:' or url[:5] == 'news:': - if self.verbose > 1: print " Not checking mailto/news URL" - return None - isint = self.inroots(url) - if not isint: - if not self.checkext: - if self.verbose > 1: print " Not checking ext link" - return None - f = self.openpage(url) - if f: - self.safeclose(f) - return None - text, nurl = self.readhtml(url) - if nurl != url: - if self.verbose > 1: - print " Redirected to", nurl - url = nurl - if text: - return Page(text, url, verbose=self.verbose, maxpage=self.maxpage) + if url[:7] == 'mailto:' or url[:5] == 'news:': + if self.verbose > 1: print " Not checking mailto/news URL" + return None + isint = self.inroots(url) + if not isint: + if not self.checkext: + if self.verbose > 1: print " Not checking ext link" + return None + f = self.openpage(url) + if f: + self.safeclose(f) + return None + text, nurl = self.readhtml(url) + if nurl != url: + if self.verbose > 1: + print " Redirected to", nurl + url = nurl + if text: + return Page(text, url, verbose=self.verbose, maxpage=self.maxpage) def readhtml(self, url): - text = None - f, url = self.openhtml(url) - if f: - text = f.read() - f.close() - return text, url + text = None + f, url = self.openhtml(url) + if f: + text = f.read() + f.close() + return text, url def openhtml(self, url): - f = self.openpage(url) - if f: - url = f.geturl() - info = f.info() - if not self.checkforhtml(info, url): - self.safeclose(f) - f = None - return f, url + f = self.openpage(url) + if f: + url = f.geturl() + info = f.info() + if not self.checkforhtml(info, url): + self.safeclose(f) + f = None + return f, url def openpage(self, url): - try: - return self.urlopener.open(url) - except IOError, msg: - msg = self.sanitize(msg) - if self.verbose > 0: - print "Error ", msg - if self.verbose > 0: - self.show(" HREF ", url, " from", self.todo[url]) - self.setbad(url, msg) - return None + try: + return self.urlopener.open(url) + except IOError, msg: + msg = self.sanitize(msg) + if self.verbose > 0: + print "Error ", msg + if self.verbose > 0: + self.show(" HREF ", url, " from", self.todo[url]) + self.setbad(url, msg) + return None def checkforhtml(self, info, url): - if info.has_key('content-type'): - ctype = string.lower(info['content-type']) - else: - if url[-1:] == "/": - return 1 - ctype, encoding = mimetypes.guess_type(url) - if ctype == 'text/html': - return 1 - else: - if self.verbose > 1: - print " Not HTML, mime type", ctype - return 0 + if info.has_key('content-type'): + ctype = string.lower(info['content-type']) + else: + if url[-1:] == "/": + return 1 + ctype, encoding = mimetypes.guess_type(url) + if ctype == 'text/html': + return 1 + else: + if self.verbose > 1: + print " Not HTML, mime type", ctype + return 0 def setgood(self, url): - if self.bad.has_key(url): - del self.bad[url] - self.changed = 1 - if self.verbose > 0: - print "(Clear previously seen error)" + if self.bad.has_key(url): + del self.bad[url] + self.changed = 1 + if self.verbose > 0: + print "(Clear previously seen error)" def setbad(self, url, msg): - if self.bad.has_key(url) and self.bad[url] == msg: - if self.verbose > 0: - print "(Seen this error before)" - return - self.bad[url] = msg - self.changed = 1 - self.markerror(url) - + if self.bad.has_key(url) and self.bad[url] == msg: + if self.verbose > 0: + print "(Seen this error before)" + return + self.bad[url] = msg + self.changed = 1 + self.markerror(url) + def markerror(self, url): - try: - origins = self.todo[url] - except KeyError: - origins = self.done[url] - for source, rawlink in origins: - triple = url, rawlink, self.bad[url] - self.seterror(source, triple) + try: + origins = self.todo[url] + except KeyError: + origins = self.done[url] + for source, rawlink in origins: + triple = url, rawlink, self.bad[url] + self.seterror(source, triple) def seterror(self, url, triple): - try: - self.errors[url].append(triple) - except KeyError: - self.errors[url] = [triple] + try: + self.errors[url].append(triple) + except KeyError: + self.errors[url] = [triple] # The following used to be toplevel functions; they have been # changed into methods so they can be overridden in subclasses. def show(self, p1, link, p2, origins): - print p1, link - i = 0 - for source, rawlink in origins: - i = i+1 - if i == 2: - p2 = ' '*len(p2) - print p2, source, - if rawlink != link: print "(%s)" % rawlink, - print + print p1, link + i = 0 + for source, rawlink in origins: + i = i+1 + if i == 2: + p2 = ' '*len(p2) + print p2, source, + if rawlink != link: print "(%s)" % rawlink, + print def sanitize(self, msg): - if isinstance(IOError, ClassType) and isinstance(msg, IOError): - # Do the other branch recursively - msg.args = self.sanitize(msg.args) - elif isinstance(msg, TupleType): - if len(msg) >= 4 and msg[0] == 'http error' and \ - isinstance(msg[3], InstanceType): - # Remove the Message instance -- it may contain - # a file object which prevents pickling. - msg = msg[:3] + msg[4:] - return msg + if isinstance(IOError, ClassType) and isinstance(msg, IOError): + # Do the other branch recursively + msg.args = self.sanitize(msg.args) + elif isinstance(msg, TupleType): + if len(msg) >= 4 and msg[0] == 'http error' and \ + isinstance(msg[3], InstanceType): + # Remove the Message instance -- it may contain + # a file object which prevents pickling. + msg = msg[:3] + msg[4:] + return msg def safeclose(self, f): - try: - url = f.geturl() - except AttributeError: - pass - else: - if url[:4] == 'ftp:' or url[:7] == 'file://': - # Apparently ftp connections don't like to be closed - # prematurely... - text = f.read() - f.close() + try: + url = f.geturl() + except AttributeError: + pass + else: + if url[:4] == 'ftp:' or url[:7] == 'file://': + # Apparently ftp connections don't like to be closed + # prematurely... + text = f.read() + f.close() def save_pickle(self, dumpfile=DUMPFILE): - if not self.changed: - if self.verbose > 0: - print - print "No need to save checkpoint" - elif not dumpfile: - if self.verbose > 0: - print "No dumpfile, won't save checkpoint" - else: - if self.verbose > 0: - print - print "Saving checkpoint to %s ..." % dumpfile - newfile = dumpfile + ".new" - f = open(newfile, "wb") - pickle.dump(self, f) - f.close() - try: - os.unlink(dumpfile) - except os.error: - pass - os.rename(newfile, dumpfile) - if self.verbose > 0: - print "Done." - return 1 + if not self.changed: + if self.verbose > 0: + print + print "No need to save checkpoint" + elif not dumpfile: + if self.verbose > 0: + print "No dumpfile, won't save checkpoint" + else: + if self.verbose > 0: + print + print "Saving checkpoint to %s ..." % dumpfile + newfile = dumpfile + ".new" + f = open(newfile, "wb") + pickle.dump(self, f) + f.close() + try: + os.unlink(dumpfile) + except os.error: + pass + os.rename(newfile, dumpfile) + if self.verbose > 0: + print "Done." + return 1 class Page: def __init__(self, text, url, verbose=VERBOSE, maxpage=MAXPAGE): - self.text = text - self.url = url - self.verbose = verbose - self.maxpage = maxpage + self.text = text + self.url = url + self.verbose = verbose + self.maxpage = maxpage def getlinkinfos(self): - size = len(self.text) - if size > self.maxpage: - if self.verbose > 0: - print "Skip huge file", self.url - print " (%.0f Kbytes)" % (size*0.001) - return [] - if self.verbose > 2: - print " Parsing", self.url, "(%d bytes)" % size - parser = MyHTMLParser(verbose=self.verbose) - parser.feed(self.text) - parser.close() - rawlinks = parser.getlinks() - base = urlparse.urljoin(self.url, parser.getbase() or "") - infos = [] - for rawlink in rawlinks: - t = urlparse.urlparse(rawlink) - t = t[:-1] + ('',) - rawlink = urlparse.urlunparse(t) - link = urlparse.urljoin(base, rawlink) - infos.append((link, rawlink)) - return infos + size = len(self.text) + if size > self.maxpage: + if self.verbose > 0: + print "Skip huge file", self.url + print " (%.0f Kbytes)" % (size*0.001) + return [] + if self.verbose > 2: + print " Parsing", self.url, "(%d bytes)" % size + parser = MyHTMLParser(verbose=self.verbose) + parser.feed(self.text) + parser.close() + rawlinks = parser.getlinks() + base = urlparse.urljoin(self.url, parser.getbase() or "") + infos = [] + for rawlink in rawlinks: + t = urlparse.urlparse(rawlink) + t = t[:-1] + ('',) + rawlink = urlparse.urlunparse(t) + link = urlparse.urljoin(base, rawlink) + infos.append((link, rawlink)) + return infos class MyStringIO(StringIO.StringIO): def __init__(self, url, info): - self.__url = url - self.__info = info - StringIO.StringIO.__init__(self) + self.__url = url + self.__info = info + StringIO.StringIO.__init__(self) def info(self): - return self.__info + return self.__info def geturl(self): - return self.__url + return self.__url class MyURLopener(urllib.FancyURLopener): @@ -590,81 +590,81 @@ class MyURLopener(urllib.FancyURLopener): http_error_default = urllib.URLopener.http_error_default def __init__(*args): - self = args[0] - apply(urllib.FancyURLopener.__init__, args) - self.addheaders = [ - ('User-agent', 'Python-webchecker/%s' % __version__), - ] + self = args[0] + apply(urllib.FancyURLopener.__init__, args) + self.addheaders = [ + ('User-agent', 'Python-webchecker/%s' % __version__), + ] def http_error_401(self, url, fp, errcode, errmsg, headers): return None def open_file(self, url): - path = urllib.url2pathname(urllib.unquote(url)) - if path[-1] != os.sep: - url = url + '/' - if os.path.isdir(path): - indexpath = os.path.join(path, "index.html") - if os.path.exists(indexpath): - return self.open_file(url + "index.html") - try: - names = os.listdir(path) - except os.error, msg: - raise IOError, msg, sys.exc_traceback - names.sort() - s = MyStringIO("file:"+url, {'content-type': 'text/html'}) - s.write('<BASE HREF="file:%s">\n' % - urllib.quote(os.path.join(path, ""))) - for name in names: - q = urllib.quote(name) - s.write('<A HREF="%s">%s</A>\n' % (q, q)) - s.seek(0) - return s - return urllib.FancyURLopener.open_file(self, path) + path = urllib.url2pathname(urllib.unquote(url)) + if path[-1] != os.sep: + url = url + '/' + if os.path.isdir(path): + indexpath = os.path.join(path, "index.html") + if os.path.exists(indexpath): + return self.open_file(url + "index.html") + try: + names = os.listdir(path) + except os.error, msg: + raise IOError, msg, sys.exc_traceback + names.sort() + s = MyStringIO("file:"+url, {'content-type': 'text/html'}) + s.write('<BASE HREF="file:%s">\n' % + urllib.quote(os.path.join(path, ""))) + for name in names: + q = urllib.quote(name) + s.write('<A HREF="%s">%s</A>\n' % (q, q)) + s.seek(0) + return s + return urllib.FancyURLopener.open_file(self, path) class MyHTMLParser(sgmllib.SGMLParser): def __init__(self, verbose=VERBOSE): - self.base = None - self.links = {} - self.myverbose = verbose - sgmllib.SGMLParser.__init__(self) + self.base = None + self.links = {} + self.myverbose = verbose + sgmllib.SGMLParser.__init__(self) def start_a(self, attributes): - self.link_attr(attributes, 'href') + self.link_attr(attributes, 'href') def end_a(self): pass def do_area(self, attributes): - self.link_attr(attributes, 'href') + self.link_attr(attributes, 'href') def do_img(self, attributes): - self.link_attr(attributes, 'src', 'lowsrc') + self.link_attr(attributes, 'src', 'lowsrc') def do_frame(self, attributes): - self.link_attr(attributes, 'src') + self.link_attr(attributes, 'src') def link_attr(self, attributes, *args): - for name, value in attributes: - if name in args: - if value: value = string.strip(value) - if value: self.links[value] = None + for name, value in attributes: + if name in args: + if value: value = string.strip(value) + if value: self.links[value] = None def do_base(self, attributes): - for name, value in attributes: - if name == 'href': - if value: value = string.strip(value) - if value: - if self.myverbose > 1: - print " Base", value - self.base = value + for name, value in attributes: + if name == 'href': + if value: value = string.strip(value) + if value: + if self.myverbose > 1: + print " Base", value + self.base = value def getlinks(self): - return self.links.keys() + return self.links.keys() def getbase(self): - return self.base + return self.base if __name__ == '__main__': |