bpo-35922: Fix RobotFileParser when robots.txt has no relevant crawl delay or request rate (GH-11791)

Co-Authored-By: Tal Einat <taleinat+github@gmail.com>
(cherry picked from commit 8047e0e1c620f69cc21f9ca48b24bf2cdd5c3668)

Co-authored-by: Rémi Lapeyre <remi.lapeyre@henki.fr>

1 files changed, 6 insertions, 2 deletions

diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py
index 7089916..c58565e 100644
--- a/Lib/urllib/robotparser.py
+++ b/Lib/urllib/robotparser.py
@@ -186,7 +186,9 @@ class RobotFileParser:
         for entry in self.entries:
             if entry.applies_to(useragent):
                 return entry.delay
-        return self.default_entry.delay
+        if self.default_entry:
+            return self.default_entry.delay
+        return None
 
     def request_rate(self, useragent):
         if not self.mtime():
@@ -194,7 +196,9 @@ class RobotFileParser:
         for entry in self.entries:
             if entry.applies_to(useragent):
                 return entry.req_rate
-        return self.default_entry.req_rate
+        if self.default_entry:
+            return self.default_entry.req_rate
+        return None
 
     def site_maps(self):
         if not self.sitemaps: