Copyright (C) 2000 Bastian Kleineidam
You can choose between two licenses when using this package:
2) PSF license for Python 2.2
The robots.txt Exclusion Protocol is implemented as specified in
http://www.robotstxt.org/norobots-rfc.txt
__all__ = ["RobotFileParser"]
RequestRate = collections.namedtuple("RequestRate", "requests seconds")
""" This class provides a set of methods to read, parse and answer
questions about a single robots.txt file.
def __init__(self, url=''):
self.default_entry = None
self.disallow_all = False
"""Returns the time the robots.txt file was last fetched.
This is useful for long-running web spiders that need to
check for new robots.txt files periodically.
"""Sets the time the robots.txt file was last fetched to the
self.last_checked = time.time()
"""Sets the URL referring to a robots.txt file."""
self.host, self.path = urllib.parse.urlparse(url)[1:3]
"""Reads the robots.txt URL and feeds it to the parser."""
f = urllib.request.urlopen(self.url)
except urllib.error.HTTPError as err:
if err.code in (401, 403):
elif err.code >= 400 and err.code < 500:
self.parse(raw.decode("utf-8").splitlines())
def _add_entry(self, entry):
if "*" in entry.useragents:
# the default entry is considered last
if self.default_entry is None:
# the first default entry wins
self.default_entry = entry
self.entries.append(entry)
"""Parse the input lines from a robots.txt file.
We allow that a user-agent: line is not preceded by
# 2: saw an allow or disallow line
# remove optional comment and strip line
line = line.split(':', 1)
line[0] = line[0].strip().lower()
line[1] = urllib.parse.unquote(line[1].strip())
if line[0] == "user-agent":
entry.useragents.append(line[1])
elif line[0] == "disallow":
entry.rulelines.append(RuleLine(line[1], False))
entry.rulelines.append(RuleLine(line[1], True))
elif line[0] == "crawl-delay":
# before trying to convert to int we need to make
# sure that robots.txt has valid syntax otherwise
if line[1].strip().isdigit():
entry.delay = int(line[1])
elif line[0] == "request-rate":
numbers = line[1].split('/')
# check if all values are sane
if (len(numbers) == 2 and numbers[0].strip().isdigit()
and numbers[1].strip().isdigit()):
entry.req_rate = RequestRate(int(numbers[0]), int(numbers[1]))
def can_fetch(self, useragent, url):
"""using the parsed robots.txt decide if useragent can fetch url"""
# Until the robots.txt file has been read or found not
# to exist, we must assume that no url is allowable.
# This prevents false positives when a user erroneously
# calls can_fetch() before calling read().
if not self.last_checked:
# search for given user agent matches
parsed_url = urllib.parse.urlparse(urllib.parse.unquote(url))
url = urllib.parse.urlunparse(('','',parsed_url.path,
parsed_url.params,parsed_url.query, parsed_url.fragment))
url = urllib.parse.quote(url)
for entry in self.entries:
if entry.applies_to(useragent):
return entry.allowance(url)
# try the default entry last
return self.default_entry.allowance(url)
# agent not found ==> access granted
def crawl_delay(self, useragent):
for entry in self.entries:
if entry.applies_to(useragent):
return self.default_entry.delay
def request_rate(self, useragent):
for entry in self.entries:
if entry.applies_to(useragent):
return self.default_entry.req_rate
if self.default_entry is not None:
entries = entries + [self.default_entry]
return '\n'.join(map(str, entries)) + '\n'
"""A rule line is a single "Allow:" (allowance==True) or "Disallow:"
(allowance==False) followed by a path."""
def __init__(self, path, allowance):
if path == '' and not allowance:
# an empty value means allow all
path = urllib.parse.urlunparse(urllib.parse.urlparse(path))
self.path = urllib.parse.quote(path)
self.allowance = allowance
def applies_to(self, filename):
return self.path == "*" or filename.startswith(self.path)
return ("Allow" if self.allowance else "Disallow") + ": " + self.path
"""An entry has one or more user-agents and zero or more rulelines"""
for agent in self.useragents:
ret.append(f"User-agent: {agent}")
if self.delay is not None:
ret.append(f"Crawl-delay: {self.delay}")
if self.req_rate is not None:
ret.append(f"Request-rate: {rate.requests}/{rate.seconds}")
ret.extend(map(str, self.rulelines))
ret.append('') # for compatibility
def applies_to(self, useragent):
"""check if this entry applies to the specified agent"""
# split the name token and make it lower case
useragent = useragent.split("/")[0].lower()
for agent in self.useragents:
# we have the catch-all agent
def allowance(self, filename):
- our agent applies to this entry
- filename is URL decoded"""
for line in self.rulelines:
if line.applies_to(filename):
It is recommended that you Edit text format, this type of Fix handles quite a lot in one request
