diff options
| author | Skip Montanaro <skip@pobox.com> | 2008-07-27 00:49:02 (GMT) |
|---|---|---|
| committer | Skip Montanaro <skip@pobox.com> | 2008-07-27 00:49:02 (GMT) |
| commit | 1ef19f0de12da730b5ba3096ab4a1d78af5185b4 (patch) | |
| tree | 474c5c3011d3fafb38929aa0e263cabe1f855186 /Lib/robotparser.py | |
| parent | 4b99e9b4790af9951b81925e28bd07850cb5c630 (diff) | |
| download | cpython-1ef19f0de12da730b5ba3096ab4a1d78af5185b4.zip cpython-1ef19f0de12da730b5ba3096ab4a1d78af5185b4.tar.gz cpython-1ef19f0de12da730b5ba3096ab4a1d78af5185b4.tar.bz2 | |
Close issue 3437 - missing state change when Allow lines are processed.
Adds test cases which use Allow: as well.
Diffstat (limited to 'Lib/robotparser.py')
| -rw-r--r-- | Lib/robotparser.py | 5 |
1 files changed, 5 insertions, 0 deletions
diff --git a/Lib/robotparser.py b/Lib/robotparser.py index f249187..447563f 100644 --- a/Lib/robotparser.py +++ b/Lib/robotparser.py @@ -76,6 +76,10 @@ class RobotFileParser: """parse the input lines from a robots.txt file. We allow that a user-agent: line is not preceded by one or more blank lines.""" + # states: + # 0: start state + # 1: saw user-agent line + # 2: saw an allow or disallow line state = 0 linenumber = 0 entry = Entry() @@ -114,6 +118,7 @@ class RobotFileParser: elif line[0] == "allow": if state != 0: entry.rulelines.append(RuleLine(line[1], True)) + state = 2 if state == 2: self.entries.append(entry) |