*j/ S SK r S SKrS SKrS SKrS SKrS/r\ R " SS5 r " S S5 r " S S5 r
" S S 5 r S
r S r
S rS
rS rg) NRobotFileParser RequestRatezrequests secondsc f \ rS rSrSS jrS rS rS rS rS r S r
S r S
r S r
S rS
rS rSrg)r c / U l 0 U l / U l S U l SU l SU l U R
U5 SU l g )NFr )entriesgroupssitemaps
default_entry disallow_all allow_allset_url last_checkedselfurls 9/opt/alt/python313/lib64/python3.13/urllib/robotparser.py__init__RobotFileParser.__init__ sB
! ! S c U R $ N)r r s r mtimeRobotFileParser.mtime( s r c 6 SS K nUR 5 U l g Nr )timer )r r s r modifiedRobotFileParser.modified1 s
IIKr c n Xl [ R R U5 SS u U l U l g )N )r urllibparseurlsplithostpathr s r r RobotFileParser.set_url9 s+ %|| 4 4S 9!A > 49r c [ R R U R 5 nUR 5 nU R UR
SS5 R 5 5 g ! [ R R aY nUR S; a SU l O'UR S: a UR S: a SU l UR 5 S nAg S nAff = f)Nzutf-8surrogateescape)i i Ti i )r$ requesturlopenr readr% decode
splitlineserror HTTPErrorcoder r
close)r frawerrs r r. RobotFileParser.read> s
L&&txx0A &&(C JJszz'+<=HHJ K ||%% xx:%$(!S SXX^!% IIKK s )A+ + C" AC C"c
U R R U5 UR HX nUR 5 nX R ; a XR U' M2 [ U R U U5 U R U' MZ g r )r append
useragentslowerr
merge_entries)r entryagents r
_add_entryRobotFileParser._add_entryL sc E"%%EKKMEKK'%* E"%24;;u3Eu%M E" &r c / nSn[ 5 nU R 5 U GH nUR S5 nUS: a US U nUR 5 nU( d M9 UR SS5 n[ U5 S:X d M\ US R 5 R
5 US' US R 5 US' US S:X aE US:X a U R U5 [ 5 nUS nUR R U5 SnM US S:X a7 US:w a. Sn UR R [ US S5 5 GM! GM$ US S :X a7 US:w a. Sn UR R [ US S
5 5 GMa GMd US S :X aG US:w a> US R 5 R 5 ( a [ US 5 Ul SnGM GM US S :X a US:w a US R S
5 n[ U5 S:X au US R 5 R 5 ( aO US R 5 R 5 ( a) [! [ US 5 [ US 5 5 Ul SnGM` GMc US S:X d GMo U R$ R US 5 GM US:w a U R U5 g g ! [ a GM f = f! [ a GM f = f)Nr #:r" z
user-agentdisallowFallowTz crawl-delayz request-rate/sitemap)Entryr findstripsplitlenr< r@ r; r: rulelinesRuleLine
ValueErrorisdigitintdelayr req_rater
) r linesr stater> linei
product_tokennumberss r r% RobotFileParser.parseU s
D #AAvBQx:: s r _find_entryRobotFileParser._find_entry sh 12 L[['')E ** * {{s##r c | U R ( a gU R ( a gU R ( d g[ R R U5 n[ R R
SS/USS Q75 n[ U5 nU( d SnUS:X a gU R U5 nUc gUR U5 $ )NFT rE rH z /robots.txt)
r r
r r$ r% r&
urlunsplit
normalize_urird allowance)r rc r
parsed_urlr> s r can_fetchRobotFileParser.can_fetch s >>
\\**3/
ll%%r2&?
12&?@C C - + =s##r c p U R 5 ( d g U R U5 nUc g UR $ r )r rd rT rb s r crawl_delayRobotFileParser.crawl_delay s2 zz|| + ={{r c p U R 5 ( d g U R U5 nUc g UR $ r )r rd rU rb s r request_rate RobotFileParser.request_rate s2 zz|| + =~~ r c > U R ( d g U R $ r )r
r s r site_mapsRobotFileParser.site_maps s }}}} r c U R nU R b XR /- nSR [ S [ [
U5 5 5 $ )Nz
)r r joinfiltermapstr)r r s r __str__RobotFileParser.__str__ sE ,, ) !3!3 44G{{6$C(9:;;r )
r
r r r r r' r r( r
r N)rg )__name__
__module__ __qualname____firstlineno__r r r r r. r@ r% rd rl ro rr ru r| __static_attributes__ r r r r sG !(?
L NG#R$$6
A-- )
&&tyy11 tyy>A--r c | U R ( a SOSS- U R - U R ( a S- $ S- $ )NAllowDisallowz: r rg )rj r( r r s r r| RuleLine.__str__ s; NN
dBTYYN ..32 3.02 3r )rj r r r( N)r~ r r r r ra r| r r r r rP rP s #* 3r rP c , \ rS rSrS rS rS rS rSrg)rJ i
c < / U l / U l S U l S U l g r )r; rO rT rU r s r r Entry.__init__ s
r c U R ( d g/ nU R H nUR SU 35 M U R b UR SU R 35 U R b7 U R nUR SUR SUR
35 U R ( a* UR [ [ U R 5 5 OUR S5 SR U5 $ )Nrg z User-agent: z
Crawl-delay: zRequest-rate: rH zAllow:
) r; r: rT rU requestssecondsrO extendrz r{ rx )r retr? rates r r|
Entry.__str__ s __E JJ eW- . % :: ! JJ tzzl3 4 == $==D JJ
a ~F G >> JJs3/ 0 JJx yy~ r c Uc SU R ; $ UR S5 S R 5 nU R H" nUS:w d M UR 5 nX!; d M" g g)Nr^ rH r TF)r; rM r< )r rc r? s r ra Entry.applies_to# sa $//) ) OOC(+113 __E|
% %
r c SnSnU R HN nUR U5 nU( d M XR: a UnUR nM2 XR:X d M9 U( a MB UR nMP U$ )NT)rO ra rj )r r
best_matchrj rX r s r rj Entry.allowance0 s\
NND)Aq>!"J $I_YY $I # r )rT rU rO r; N) r~ r r r r r| ra rj r r r r rJ rJ
s " r rJ c z [ R R U SS9n[ R R USS9$ )Nr+ )errors)r$ r% unquotequote)r( unquoteds r normalizer B s7 ||##D1B#CH << h/@ AAr c U R S5 u pn[ U 5 n U( a! [ R " SS U5 nU SU- -
n U $ )N?z[^=&]+c [ U S 5 $ r r r s r normalize_uri..J s IadOr ) partitionr r r r( sepquerys r ri ri F sG ~~c*Du T?D
y";UC e Kr c U R S5 u pn[ R " SS U 5 n U( a! [ R " SS U5 nU SU- -
n U $ )Nr z[^*$]+c [ U S 5 $ r r r s r r #normalize_pattern..P s y1r z[^=&*$]+c [ U S 5 $ r r r s r r r R s i!or )r r r r s r r r N sQ ~~c*Du
66) 6 =D
{$=uE e Kr c [ [ [ R U R S5 5 5 n[ S[
U5 S-
5 H
nSX S3X' M SUS 3US' SR U5 $ )Nr^ r" z(?>.*?)z.*r rg )listrz r escaperM rangerN rx )r( partsrY s r r r V sl RYY
30 1E
1c%jl
#EH:Q' $U2YK E"I
775>r c [ 5 n[ [ [ UR 5 R
U R 5 5 Ul U R UR - Ul UR c U R O UR Ul UR c U R Ul U$ UR Ul U$ r ) rJ r ry setr; __contains__rO rT rU )e1e2r> s r r= r= ] s GEF3r}}#5#B#BBMM RSEllR\\1EO hh ."((BHHEK$&KK$7R[[EN L >@[[EN Lr ) collectionsr urllib.errorr$ urllib.parseurllib.request__all__
namedtupler r rP rJ r ri r r r= r r r r sn
$$]4FG C< C