diff options
| author | Serhiy Storchaka <storchaka@gmail.com> | 2018-05-14 22:09:47 (GMT) |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2018-05-14 22:09:47 (GMT) |
| commit | 861d38443d4b85cdc7b87afc4adee55f51c2f4b3 (patch) | |
| tree | 9e702bbf51583e4740b44af2a1d113ba007d811d | |
| parent | 84fc6c59cf286fc4e6b3a700c6db36ecc2bff92b (diff) | |
| download | cpython-861d38443d4b85cdc7b87afc4adee55f51c2f4b3.zip cpython-861d38443d4b85cdc7b87afc4adee55f51c2f4b3.tar.gz cpython-861d38443d4b85cdc7b87afc4adee55f51c2f4b3.tar.bz2 | |
[2.7] bpo-32861: robotparser fix incomplete __str__ methods. (GH-5711) (GH-6795) (GH-6817)
The robotparser's __str__ representation now includes wildcard
entries.
(cherry picked from commit c3fa1f2b93fa4bf96a8aadc74ee196384cefa31e)
Co-authored-by: Michael Lazar <lazar.michael22@gmail.com>.
| -rw-r--r-- | Lib/robotparser.py | 5 | ||||
| -rw-r--r-- | Lib/test/test_robotparser.py | 26 | ||||
| -rw-r--r-- | Misc/ACKS | 1 | ||||
| -rw-r--r-- | Misc/NEWS.d/next/Library/2018-04-02-20-44-54.bpo-32861.HeBjzN.rst | 3 |
4 files changed, 34 insertions, 1 deletions
diff --git a/Lib/robotparser.py b/Lib/robotparser.py index a7137a3..4e13f7f 100644 --- a/Lib/robotparser.py +++ b/Lib/robotparser.py @@ -160,7 +160,10 @@ class RobotFileParser: def __str__(self): - return ''.join([str(entry) + "\n" for entry in self.entries]) + entries = self.entries + if self.default_entry is not None: + entries = entries + [self.default_entry] + return '\n'.join(map(str, entries)) + '\n' class RuleLine: diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index 8ed5d89..ba7ccf8 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -136,6 +136,31 @@ Disallow: /cyberworld/map/ bad = ['/cyberworld/map/index.html'] +class StringFormattingTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +User-agent: * +Crawl-delay: 1 +Request-rate: 3/15 +Disallow: /cyberworld/map/ # This is an infinite virtual URL space + +# Cybermapper knows where to go. +User-agent: cybermapper +Disallow: /some/path + """ + + expected_output = """\ +User-agent: cybermapper +Disallow: /some/path + +User-agent: * +Disallow: /cyberworld/map/ + +""" + + def test_string_formatting(self): + self.assertEqual(str(self.parser), self.expected_output) + + class RobotHandler(BaseHTTPRequestHandler): def do_GET(self): @@ -226,6 +251,7 @@ def test_main(): UseFirstUserAgentWildcardTest, EmptyQueryStringTest, DefaultEntryTest, + StringFormattingTest, PasswordProtectedSiteTestCase, NetworkTestCase) @@ -807,6 +807,7 @@ Ben Laurie Simon Law Julia Lawall Chris Lawrence +Michael Lazar Brian Leair Mathieu Leduc-Hamel Amandine Lee diff --git a/Misc/NEWS.d/next/Library/2018-04-02-20-44-54.bpo-32861.HeBjzN.rst b/Misc/NEWS.d/next/Library/2018-04-02-20-44-54.bpo-32861.HeBjzN.rst new file mode 100644 index 0000000..13defbb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-04-02-20-44-54.bpo-32861.HeBjzN.rst @@ -0,0 +1,3 @@ +The urllib.robotparser's ``__str__`` representation now includes wildcard +entries and the "Crawl-delay" and "Request-rate" fields. Patch by +Michael Lazar. |