scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware

class documentation

class RobotsTxtMiddleware:

View In Hierarchy

Undocumented

Class Method from​_crawler Undocumented
Constant DOWNLOAD​_PRIORITY Undocumented
Method __init__ Undocumented
Method ​_logerror Undocumented
Method ​_parse​_robots Undocumented
Method ​_robots​_error Undocumented
Method process​_request Undocumented
Method process​_request​_2 Undocumented
Method robot​_parser Undocumented
Instance Variable ​_default​_useragent Undocumented
Instance Variable ​_parserimpl Undocumented
Instance Variable ​_parsers Undocumented
Instance Variable ​_robotstxt​_useragent Undocumented
Instance Variable crawler Undocumented
@classmethod
def from_crawler(cls, crawler):

Undocumented

DOWNLOAD_PRIORITY: int =

Undocumented

Value
1000
def __init__(self, crawler):

Undocumented

def _logerror(self, failure, request, spider):

Undocumented

def _parse_robots(self, response, netloc, spider):

Undocumented

def _robots_error(self, failure, netloc):

Undocumented

def process_request(self, request, spider):

Undocumented

def process_request_2(self, rp, request, spider):

Undocumented

def robot_parser(self, request, spider):

Undocumented

_default_useragent =

Undocumented

_parserimpl =

Undocumented

_parsers: dict =

Undocumented

_robotstxt_useragent =

Undocumented

crawler =

Undocumented