summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rwxr-xr-xTools/webchecker/webchecker.py18
1 files changed, 13 insertions, 5 deletions
diff --git a/Tools/webchecker/webchecker.py b/Tools/webchecker/webchecker.py
index 6b6fbf5..2ec9b61 100755
--- a/Tools/webchecker/webchecker.py
+++ b/Tools/webchecker/webchecker.py
@@ -93,6 +93,8 @@ rooturl -- URL to start checking
"""
+__version__ = "0.1"
+
import sys
import os
@@ -135,7 +137,6 @@ def main():
except getopt.error, msg:
sys.stdout = sys.stderr
print msg
- print __doc__ % globals()
sys.exit(2)
for o, a in opts:
if o == '-R':
@@ -151,6 +152,9 @@ def main():
if o == '-v':
verbose = verbose + 1
+ if verbose:
+ print AGENTNAME, "version", __version__
+
if restart:
if verbose > 0:
print "Loading checkpoint from %s ..." % dumpfile
@@ -234,13 +238,17 @@ class Checker:
self.addrobot(root)
def addrobot(self, root):
+ url = urlparse.urljoin(root, "/robots.txt")
self.robots[root] = rp = robotparser.RobotFileParser()
- if verbose > 3:
- print "Parsing robots.txt file"
+ if verbose > 2:
+ print "Parsing", url
rp.debug = 1
- url = urlparse.urljoin(root, "/robots.txt")
rp.set_url(url)
- rp.read()
+ try:
+ rp.read()
+ except IOError, msg:
+ if verbose > 1:
+ print "I/O error parsing", url, ":", msg
def run(self):
while self.todo: