From 1a413136843f17ede1d3f87f9bab23a07748312c Mon Sep 17 00:00:00 2001 From: Skip Montanaro Date: Tue, 28 Aug 2007 23:22:52 +0000 Subject: fixes 813986 --- Lib/robotparser.py | 5 +++++ Lib/test/test_robotparser.py | 11 +++++++++++ 2 files changed, 16 insertions(+) diff --git a/Lib/robotparser.py b/Lib/robotparser.py index 0a2ede1..52ab348 100644 --- a/Lib/robotparser.py +++ b/Lib/robotparser.py @@ -230,6 +230,11 @@ class URLopener(urllib.FancyURLopener): urllib.FancyURLopener.__init__(self, *args) self.errcode = 200 + def prompt_user_passwd(self, host, realm): + ## If robots.txt file is accessible only with a password, + ## we act as if the file wasn't there. + return None, None + def http_error_default(self, url, fp, errcode, errmsg, headers): self.errcode = errcode return urllib.FancyURLopener.http_error_default(self, url, fp, errcode, diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index 666d00a..9163ef8 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -134,8 +134,19 @@ bad = [] # Bug report says "/" should be denied, but that is not in the RFC RobotTest(7, doc, good, bad) +class TestCase(unittest.TestCase): + def runTest(self): + test_support.requires('network') + # whole site is password-protected. + url = 'http://mueblesmoraleda.com' + parser = robotparser.RobotFileParser() + parser.set_url(url) + parser.read() + self.assertEqual(parser.can_fetch("*", url+"/robots.txt"), False) + def test_main(): test_support.run_unittest(tests) + TestCase().run() if __name__=='__main__': test_support.Verbose = 1 -- cgit v0.12