From bd08a0af2d88c590ede762102bd42da3437e9980 Mon Sep 17 00:00:00 2001
From: Michael Lazar <lazar.michael22@gmail.com>
Date: Mon, 14 May 2018 10:10:41 -0400
Subject: bpo-32861: urllib.robotparser fix incomplete __str__ methods.
 (GH-5711)

The urllib.robotparser's __str__ representation now includes wildcard
entries and the "Crawl-delay" and "Request-rate" fields. Also removes extra
newlines that were being appended to the end of the string.
---
 Lib/test/test_robotparser.py                       | 26 ++++++++++++++++++++++
 Lib/urllib/robotparser.py                          | 17 +++++++++-----
 .../2018-04-02-20-44-54.bpo-32861.HeBjzN.rst       |  4 ++++
 3 files changed, 42 insertions(+), 5 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Library/2018-04-02-20-44-54.bpo-32861.HeBjzN.rst

diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py
index 75198b7..bee8d23 100644
--- a/Lib/test/test_robotparser.py
+++ b/Lib/test/test_robotparser.py
@@ -246,6 +246,32 @@ Disallow: /cyberworld/map/
     bad = ['/cyberworld/map/index.html']
 
 
+class StringFormattingTest(BaseRobotTest, unittest.TestCase):
+    robots_txt = """\
+User-agent: *
+Crawl-delay: 1
+Request-rate: 3/15
+Disallow: /cyberworld/map/ # This is an infinite virtual URL space
+
+# Cybermapper knows where to go.
+User-agent: cybermapper
+Disallow: /some/path
+    """
+
+    expected_output = """\
+User-agent: cybermapper
+Disallow: /some/path
+
+User-agent: *
+Crawl-delay: 1
+Request-rate: 3/15
+Disallow: /cyberworld/map/\
+"""
+
+    def test_string_formatting(self):
+        self.assertEqual(str(self.parser), self.expected_output)
+
+
 class RobotHandler(BaseHTTPRequestHandler):
 
     def do_GET(self):
diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py
index daac29c..92e4efe 100644
--- a/Lib/urllib/robotparser.py
+++ b/Lib/urllib/robotparser.py
@@ -190,7 +190,10 @@ class RobotFileParser:
         return self.default_entry.req_rate
 
     def __str__(self):
-        return ''.join([str(entry) + "\n" for entry in self.entries])
+        entries = self.entries
+        if self.default_entry is not None:
+            entries = entries + [self.default_entry]
+        return '\n\n'.join(map(str, entries))
 
 
 class RuleLine:
@@ -222,10 +225,14 @@ class Entry:
     def __str__(self):
         ret = []
         for agent in self.useragents:
-            ret.extend(["User-agent: ", agent, "\n"])
-        for line in self.rulelines:
-            ret.extend([str(line), "\n"])
-        return ''.join(ret)
+            ret.append(f"User-agent: {agent}")
+        if self.delay is not None:
+            ret.append(f"Crawl-delay: {self.delay}")
+        if self.req_rate is not None:
+            rate = self.req_rate
+            ret.append(f"Request-rate: {rate.requests}/{rate.seconds}")
+        ret.extend(map(str, self.rulelines))
+        return '\n'.join(ret)
 
     def applies_to(self, useragent):
         """check if this entry applies to the specified agent"""
diff --git a/Misc/NEWS.d/next/Library/2018-04-02-20-44-54.bpo-32861.HeBjzN.rst b/Misc/NEWS.d/next/Library/2018-04-02-20-44-54.bpo-32861.HeBjzN.rst
new file mode 100644
index 0000000..2bb24d7
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2018-04-02-20-44-54.bpo-32861.HeBjzN.rst
@@ -0,0 +1,4 @@
+The urllib.robotparser's ``__str__`` representation now includes wildcard
+entries and the "Crawl-delay" and "Request-rate" fields. Also removes extra
+newlines that were being appended to the end of the string. Patch by
+Michael Lazar.
-- 
cgit v0.12