e f4 @ sd d Z d d l Z d d l Z d g Z Gd d d Z Gd d d Z Gd d d Z d S) a% robotparser.py
Copyright (C) 2000 Bastian Kleineidam
You can choose between two licenses when using this package:
1) GNU GPLv2
2) PSF license for Python 2.2
The robots.txt Exclusion Protocol is implemented as specified in
http://www.robotstxt.org/norobots-rfc.txt
NRobotFileParserc @ s e Z d Z d Z d d d Z d d Z d d Z d d
Z d d Z d
d Z d d Z
d d Z d d Z d S)r zs This class provides a set of methods to read, parse and answer
questions about a single robots.txt file.
c C s> g | _ d | _ d | _ d | _ | j | d | _ d S)NFr )entries
default_entry disallow_all allow_allset_url last_checked)selfurl r 7/opt/alt/python34/lib64/python3.4/urllib/robotparser.py__init__ s
zRobotFileParser.__init__c C s | j S)zReturns the time the robots.txt file was last fetched.
This is useful for long-running web spiders that need to
check for new robots.txt files periodically.
)r )r
r r r
mtime s zRobotFileParser.mtimec C s d d l } | j | _ d S)zYSets the time the robots.txt file was last fetched to the
current time.
r N)timer )r
r r r r
modified( s zRobotFileParser.modifiedc C s5 | | _ t j j | d d \ | _ | _ d S)z,Sets the URL referring to a robots.txt file. N)r urllibparseurlparsehostpath)r
r r r r
r 0 s zRobotFileParser.set_urlc C s y t j j | j } Wnp t j j k
r } zJ | j d k rO d | _ n* | j d k ry | j d k ry d | _ n WYd d } ~ Xn) X| j } | j
| j d j d S) z4Reads the robots.txt URL and feeds it to the parser. Ti i Nzutf-8)r r )
r ZrequestZurlopenr errorZ HTTPErrorcoder r readr decode
splitlines)r
ferrrawr r r
r 5 s zRobotFileParser.readc C sA d | j k r- | j d k r= | | _ q= n | j j | d S)N*)
useragentsr r append)r
entryr r r
_add_entryB s zRobotFileParser._add_entryc C s d } t } | j x| D]} | sx | d k rJ t } d } qx | d k rx | j | t } d } qx n | j d } | d k r | d | } n | j } | s q n | j d d } t | d k r | d j j | d s z+RobotFileParser.__str__..)joinr )r
r r r
__str__ s zRobotFileParser.__str__N)
__name__
__module__ __qualname____doc__r r r r r r' r r> rD r r r r
r s
4 c @ s: e Z d Z d Z d d Z d d Z d d Z d S) r3 zoA rule line is a single "Allow:" (allowance==True) or "Disallow:"
(allowance==False) followed by a path.c C s\ | d k r | r d } n t j j t j j | } t j j | | _ | | _ d S)Nr T)r r r9 r r: r r< )r
r r< r r r
r s
zRuleLine.__init__c C s | j d k p | j | j S)Nr# )r
startswith)r
filenamer r r
r; s zRuleLine.applies_toc C s | j r d p d d | j S)NZAllowZDisallowz: )r< r )r
r r r
rD s zRuleLine.__str__N)rE rF rG rH r r; rD r r r r
r3 s r3 c @ sF e Z d Z d Z d d Z d d Z d d Z d d Z d
S) r+ z?An entry has one or more user-agents and zero or more rulelinesc C s g | _ g | _ d S)N)r$ r2 )r
r r r
r s zEntry.__init__c C sj g } x' | j D] } | j d | d g q Wx* | j D] } | j t | d g q: Wd j | S)Nz User-agent: r? r )r$ extendr2 r@ rC )r
Zretagentr6 r r r
rD s z
Entry.__str__c C s] | j d d j } x= | j D]2 } | d k r9 d S| j } | | k r# d Sq# Wd S)z2check if this entry applies to the specified agentr8 r r# TF)r. r0 r$ )r
r= rL r r r
r; s zEntry.applies_toc C s. x' | j D] } | j | r
| j Sq
Wd S)zZPreconditions:
- our agent applies to this entry
- filename is URL decodedT)r2 r; r< )r
rJ r6 r r r
r< s zEntry.allowanceN)rE rF rG rH r rD r; r< r r r r
r+ s
r+ )rH Z urllib.parser Zurllib.request__all__r r3 r+ r r r r
s