diff options
| author | Georg Brandl <georg@python.org> | 2010-08-01 20:59:03 +0000 |
|---|---|---|
| committer | Georg Brandl <georg@python.org> | 2010-08-01 20:59:03 +0000 |
| commit | 2bd953e2913fe963be53f91edfd9950a41ac53e3 (patch) | |
| tree | e998dbc6dbb58b969c63ba4dc5439d6645143219 /Lib/robotparser.py | |
| parent | e27d044769166e47310345e7d538f54e1dac04d1 (diff) | |
| download | cpython-git-2bd953e2913fe963be53f91edfd9950a41ac53e3.tar.gz | |
Merged revisions 83238 via svnmerge from
svn+ssh://pythondev@svn.python.org/python/branches/py3k
........
r83238 | georg.brandl | 2010-07-29 19:55:01 +0200 (Do, 29 Jul 2010) | 1 line
#4108: the first default entry (User-agent: *) wins.
........
Diffstat (limited to 'Lib/robotparser.py')
| -rw-r--r-- | Lib/robotparser.py | 6 |
1 files changed, 4 insertions, 2 deletions
diff --git a/Lib/robotparser.py b/Lib/robotparser.py index 730426f6ae..1722863d14 100644 --- a/Lib/robotparser.py +++ b/Lib/robotparser.py @@ -68,7 +68,9 @@ class RobotFileParser: def _add_entry(self, entry): if "*" in entry.useragents: # the default entry is considered last - self.default_entry = entry + if self.default_entry is None: + # the first default entry wins + self.default_entry = entry else: self.entries.append(entry) @@ -120,7 +122,7 @@ class RobotFileParser: entry.rulelines.append(RuleLine(line[1], True)) state = 2 if state == 2: - self.entries.append(entry) + self._add_entry(entry) def can_fetch(self, useragent, url): |