3
Klf @ s< d Z dZddlZddlZddlZddlZddlZddlZyejej fZ
W n e k
rd e e
fZ
Y nX ejdZG dd deZG dd deZG d d
d
eZG d d d eZG d
d dZdd Zdd Zdd Zdd Zdd Zdd Z G dd d eZ ddd d d e ej d dddf
d d Z d%d!d"Z!d#d$ Z"e"Z#dS )&z3.11z3.10 Nz^[a-zA-Z0-9_]+$c @ s e Zd Zdd ZdS )LexErrorc C s |f| _ || _d S )N)argstext)selfmessages r /usr/lib/python3.6/lex.py__init__: s zLexError.__init__N)__name__
__module__ __qualname__r
r r r r r 9 s r c @ s e Zd Zdd Zdd ZdS )LexTokenc C s d| j | j| j| jf S )NzLexToken(%s,%r,%d,%d))typevaluelinenolexpos)r r r r __str__A s zLexToken.__str__c C s t | S )N)str)r r r r __repr__D s zLexToken.__repr__N)r r r
r r r r r r r @ s r c @ s4 e Zd Zdd Zdd Zdd Zdd ZeZeZd S )
PlyLoggerc C s
|| _ d S )N)f)r r r r r r
L s zPlyLogger.__init__c O s | j j|| d d S )N
)r write)r msgr kwargsr r r criticalO s zPlyLogger.criticalc O s | j jd|| d d S )Nz WARNING: r )r r )r r r r r r r warningR s zPlyLogger.warningc O s | j jd|| d d S )NzERROR: r )r r )r r r r r r r errorU s zPlyLogger.errorN) r r r
r
r r r infodebugr r r r r K s r c @ s e Zd Zdd Zdd ZdS )
NullLoggerc C s | S )Nr )r namer r r __getattribute__^ s zNullLogger.__getattribute__c O s | S )Nr )r r r r r r __call__a s zNullLogger.__call__N)r r r
r# r$ r r r r r! ] s r! c @ s| e Zd Zdd Zd ddZd ddZd d
Zd d Zd
d Zdd Z dd Z
dd Z dd Z dd Z
dd Zdd ZeZdS ) Lexerc C s d | _ d | _i | _i | _i | _d| _g | _d | _i | _i | _ i | _
d| _ d | _ d| _
d| _d | _d | _d | _d| _d| _d | _d| _d| _d S )NINITIALr F)lexre lexretext
lexstaterelexstateretextlexstaterenameslexstate
lexstatestack lexstateinfolexstateignorelexstateerrorf lexstateeoff
lexreflagslexdatar lexlen lexerrorflexeoff lextokens lexignore lexliterals lexmoduler lexoptimize)r r r r r
t s. zLexer.__init__Nc C s t j | }|ri }x| jj D ]\}}g }x\|D ]T\}}g } xF|D ]>}
|
sV|
d rb| j|
qB| jt||
d j|
d f qBW q0W |j|| f |||< q W ||_i |_x(| jj D ]\}} t|| j|j|< qW ||_|S )Nr r( )copyr+ itemsappendgetattrr r2 r< ) r objectcZnewtabkeyZritemZnewreZcreZfindexZ newfindexr efr r r clone s(
& z Lexer.cloner' c C s t |tjrtd|jdd }tjj||d }t|d}|j d|t
f |j dt t |j dt t
t| j |j d t t| j |j d
t | j |j d t | j i }xb| jj D ]T\}}g } x>t|| j| | j| D ]"\\}
} } }
| j| t| |
f qW | ||< qW |j d t | |j d
t | j i }x,| j j D ] \}}|rt|j nd ||< q`W |j dt | i }x,| j j D ] \}}|r|j nd ||< qW |j dt | W d Q R X d S )Nz&Won't overwrite existing lextab module.r( z.pywzJ# %s.py. This file automatically created by PLY (version %s). Don't edit!
z_tabversion = %s
z_lextokens = set(%s)
z_lexreflags = %s
z_lexliterals = %s
z_lexstateinfo = %s
z_lexstatere = %s
z_lexstateignore = %s
z_lexstateerrorf = %s
z_lexstateeoff = %s
)
isinstancetypes
ModuleTypeIOErrorsplitospathjoinopenr __version__repr__tabversion__tuplesortedr9 intr4 r; r0 r+ r? zipr, r- r@ _funcs_to_namesr1 r2 r r3 )r lextab outputdirZ
basetabmodulefilenameZtfZtabre statenamelretitempatfuncZretextrenamesZtaberrrE Ztabeofr r r writetab s6 ( zLexer.writetabc C sR t |tjr|}ntd| tj| }t|ddtkr@td|j | _
|j | _ |j
| _| j
t| jB | _|j| _|j| _i | _i | _xb|jj D ]T\}}g }g }x.|D ]&\}} |jtj||j t | |f qW || j|< || j|< qW i | _ x$|j j D ]\}}
||
| j |< qW i | _ x&|j j D ]\}}
||
| j |< q(W | j!d d S )Nz import %sZ _tabversionz0.0zInconsistent PLY versionr& )"rJ rK rL execsysmodulesrA rU ImportErrorZ
_lextokensr9 Z _lexreflagsr4 Z _lexliteralsr; set
lextokens_allZ
_lexstateinfor0 Z_lexstateignorer1 r+ r, Z _lexstaterer? r@ recompile_names_to_funcsr2 Z_lexstateerrorfr3 Z
_lexstateeoffbegin) r Ztabfilefdictr[ r^ r_ r` Ztxtitemra Z func_namerE r r r readtab s8
"
z
Lexer.readtabc C s8 |d d }t |ts td|| _d| _t|| _d S )Nr( zExpected a stringr )rJ StringTypes
ValueErrorr5 r lenr6 )r r rC r r r input s
z Lexer.inputc C sd || j krtd| j | | _| j| | _| jj|d| _| jj|d | _ | j
j|d | _ || _ d S )NzUndefined stater' )
r+ rr r) r, r* r1 getr: r2 r7 r3 r8 r. )r stater r r rn s
z Lexer.beginc C s | j j| j | j| d S )N)r/ r@ r. rn )r rv r r r
push_state s zLexer.push_statec C s | j | jj d S )N)rn r/ pop)r r r r pop_state s zLexer.pop_statec C s | j S )N)r. )r r r r
current_state! s zLexer.current_statec C s | j |7 _ d S )N)r )r nr r r skip' s z
Lexer.skipc C s~ | j }| j}| j}| j}x||k r|| |kr<|d7 }q x| jD ]\}}|j||}|s`qFt }|j |_| j |_ ||_ |j
} || \}
|_ |
s|j r|j | _ |S |j }P |j }| |_
|| _|| _ |
|} | s| j }| j}P | js(| j | jkr(td|
jj|
jj|
j| j f ||d | S W || | jkrrt }|| |_| j |_ |j|_ ||_ |d | _ |S | jrt }| j|d |_| j |_ d|_ | |_
||_ || _ | j|} || j krtd|| ||d | j }| sq | S || _ td|| |f ||d q W | jr\t }d|_ d|_| j |_ ||_ | |_
|| _ | j|} | S |d | _ | jd krztdd S ) Nr( z4%s:%d: Rule '%s' returned an unknown token type '%s'r z&Scanning error. Illegal character '%s'z"Illegal character '%s' at index %deofr' z"No input string given with input())r r6 r: r5 r) matchr groupr r lastindexr endlexerZlexmatchr= rj r __code__ co_filenameco_firstlinenor r; r7 r8 RuntimeError) r r r6 r: r5 r) lexindexfuncmtokirb Znewtokr r r token1 s
"
z Lexer.tokenc C s | S )Nr )r r r r __iter__ s zLexer.__iter__c C s | j }|d krt|S )N)r
StopIteration)r tr r r next s z
Lexer.next)N)r' )r r r
r
rF rd rp rt rn rw ry rz r| r r r __next__r r r r r% s s
%(
nr% c C s t | d| jS )Nregex)rA __doc__)rb r r r
_get_regex s r c C s0 t j| }|jj }|j|jkr,|j|j |S )N)rf _getframe f_globalsr> f_localsupdate)Zlevelsr ldictr r r get_caller_module_dict s
r c C sJ g }x@t | |D ]2\}}|r8|d r8|j||d f q|j| qW |S )Nr r( )rY r@ )Zfunclistnamelistresultr r" r r r rZ s rZ c C sH g }x>| D ]6}|r6|d r6|j ||d |d f q
|j | q
W |S )Nr r( )r@ )r ro r r{ r r r rm s
rm c C sd | sg S dj | }ytj||}d gt|jj d }|d d }x|jj D ]z\}} |j|d }
t|
t j
t j fkr|
|| f|| < ||| < qP|
d k rP||| < |j ddkrd|| < qPd || f|| < qPW ||fg|g|gfS t
k
r^ tt| d } | dkrd} t| d | |||\} }
}t| | d |||\}}}| | |
| || fS X d S )N|r( ignore_r )NN)rQ rk rl max
groupindexvaluesr? ru r rK FunctionType
MethodTypefind ExceptionrX rs _form_master_re)Zrelistreflagsr toknamesr r) r Z
lexindexnamesr r Zhandler Zllistr_ ZlnamesZrlistZrreZrnamesr r r r s2
r c C s | j d}x0t|dd dD ]\}}||kr |dkr P q W |dkrVt|d| }nd}d|krjt|}dj||d }||fS )N_r( ANYr& )r& )rN enumeraterV rQ )r namespartsr partstatesZ tokennamer r r _statetoken s
r c @ sf e Zd ZdddZdd Zdd Zd d
Zd d Zd
d Zdd Z dd Z
dd Z dd Z dd Z
dS ) LexerReflectNr c C sL || _ d | _g | _|| _ddi| _t | _d| _|d krBtt j
n|| _ d S )Nr& inclusiveF) r Z
error_functokensr stateinfori rg r r rf stderrlog)r r r r r r r r
. s
zLexerReflect.__init__c C s$ | j | j | j | j d S )N)
get_tokens get_literals
get_states get_rules)r r r r get_all9 s zLexerReflect.get_allc C s | j | j | j | jS )N)validate_tokensvalidate_literalsvalidate_rulesr )r r r r validate_all@ s zLexerReflect.validate_allc C sp | j jdd }|s(| jjd d| _d S t|ttfsL| jjd d| _d S |sf| jjd d| _d S || _d S )Nr zNo token list is definedTz tokens must be a list or tupleztokens is empty)r ru r r rJ listrV r )r r r r r r G s zLexerReflect.get_tokensc C sT i }xJ| j D ]@}tj|s.| jjd| d| _||krD| jjd| d||< q W d S )NzBad token name '%s'TzToken '%s' multiply definedr( )r _is_identifierr~ r r r )r Z terminalsr{ r r r r [ s
z LexerReflect.validate_tokensc C s | j jdd| _| js d| _d S )Nliteralsr' )r ru r )r r r r r f s zLexerReflect.get_literalsc C sp yDx>| j D ]4}t|t s&t|dkr
| jjdt| d| _q
W W n& tk
rj | jjd d| _Y nX d S )Nr( z.Invalid literal %s. Must be a single characterTzIInvalid literals specification. literals must be a sequence of characters)r rJ rq rs r r rT TypeError)r rC r r r r l s z LexerReflect.validate_literalsc C s | j jdd | _| jrt| jttfs:| jjd d| _nx| jD ]}t|t s^t|dkrx| jjdt | d| _qB|\}}t|t
s| jjdt | d| _qB|dkp|dks| jjd | d| _qB|| j kr| jjd
| d| _qB|| j |< qBW d S ) Nr z)states must be defined as a tuple or listTr zMInvalid state specifier %s. Must be a tuple (statename,'exclusive|inclusive')z State name %s must be a stringr exclusivez:State type for state %s must be 'inclusive' or 'exclusive'zState '%s' already defined) r ru r rJ rV r r r rs rT rq r )r r r" Z statetyper r r r w s0
zLexerReflect.get_statesc C sR dd | j D }i | _i | _i | _i | _i | _i | _x"| jD ]}g | j|< g | j|< q s z*LexerReflect.get_rules..r z+No rules of the form t_rulename are definedTr$ r r} ignorez,%s:%d: Rule '%s' must be defined as a string\z#%s contains a literal backslash '\'z'Rule '%s' must be defined as a functionz&%s not defined as a function or stringc S s | d j jS )Nr( )r r )xr r r s z(LexerReflect.get_rules..)rD c S s t | d S )Nr( )rs )r r r r r s )rD reverse)r r funcsymstrsymr errorfeoffr rs r r r hasattrr r r r r@ rJ rq r r sort) r Ztsymbolsr r r r toknamelinefiler r r r sb
zLexerReflect.get_rulesc C s xp| j D ]d}x|| j| D ]l\}}|jj}|jj}tj|}| jj| | j | }t
|t j rjd}nd}|jj
} | |kr| jjd|||j d| _q | |k r| jjd|||j d| _q t|s| jjd|||j d| _q yDtjd|t|f | j}
|
jdr$| jjd |||j d| _W q tjk
r } zD| jjd
|||j| d t|krt| jjd |||j d| _W Y d d } ~ X q X q W x
| j| D ]\} }
| j | }|d
kr| jjd| d| _q|| jkr|jddk r| jjd| | d| _qy:tjd| |
f | j}
|
jdr@| jjd| d| _W nT tjk
r } z4| jjd| | d |
kr| jjd| d| _W Y d d } ~ X nX qW | j| r| j| r| jjd| d| _| jj|d }|r
|}|jj}|jj}tj|}| jj| t
|t j r d}nd}|jj
} | |krN| jjd|||j d| _| |k r
| jjd|||j d| _q
W x| jD ]}| j| q|W d S )Nr r( z'%s:%d: Rule '%s' has too many argumentsTz%%s:%d: Rule '%s' requires an argumentz2%s:%d: No regular expression defined for rule '%s'z
(?P<%s>%s)r' z<%s:%d: Regular expression for rule '%s' matches empty stringz3%s:%d: Invalid regular expression for rule '%s'. %s#z6%s:%d. Make sure '#' in rule '%s' is escaped with '\#'r z'Rule '%s' must be defined as a functionr r z-Rule '%s' defined for an unspecified token %sz5Regular expression for rule '%s' matches empty stringz,Invalid regular expression for rule '%s'. %sz/Make sure '#' in rule '%s' is escaped with '\#'z No rules defined for state '%s') r r r r r inspectZ getmodulerg addr rJ rK r co_argcountr r r r rk rl r r~ r r r r ru validate_module)r rv fnamer r r moduler ZreqargsnargsrC er" rZefuncr r r r s
zLexerReflect.validate_rulesc C s yt j|\}}W n tk
r& d S X tjd}tjd}i }|d7 }xv|D ]n}|j|}|sj|j|}|r|jd} |j| }
|
s||| < n$t j|} | j j
d| || |
d| _
|d7 }qNW d S )Nz \s*def\s+(t_[a-zA-Z_0-9]*)\(z\s*(t_[a-zA-Z_0-9]*)\s*=r( z7%s:%d: Rule %s redefined. Previously defined on line %dT) r ZgetsourcelinesrM rk rl r~ r ru Z
getsourcefiler r ) r r linesZlinenZfreZsreZ counthashr r r" prevr] r r r r ? s*
z LexerReflect.validate_module)Nr )r r r
r
r r r r r r r r r r r r r r r - s
Bgr Fr[ c
# s |d kr d}d }
ddi} t } || _| d kr6ttj} |rL|d krLttj}|rT| r fddt D }
t|
}
d|
krtj|
d j|
d< nt d}
|
j
d }|rt |t rd
|kr|d
| }t
|
| |d }|j |s|j rtd |o|r4y | j||
| ja| ja| a| S tk
r2 Y nX |rd|jd
|j |jd|j |jd|j t | _x |jD ]}| jj | qtW t |jt t frt |jd j |j| _!n|j| _!| jt| j!B | _"|j} i }x| D ]}g }xH|j#| D ]:\}}|j$d|t%|f |r|jd|t%|| qW x@|j&| D ]2\}}|j$d||f |r@|jd||| q@W |||< qW |r|jd xt|D ]l}t'|| ||
|j(\}}}|| j)|< || j*|< || j+|< |rx&t,|D ]\}} |jd||| qW qW xl| j- D ]`\}} |dkr| dkr| j)| j.| j)d | j*| j.| j*d | j+| j.| j+d qW | | _/| j)d | _0| j*d | _1|| _2|j3| _4| j4j
dd| _5|j6| _7|j6j
dd | _8| j8s| j9d |j:| _;|j:j
dd | _r|j} nNd
|kr|
d } n:|j?d
} d
j | d d }!t@d|! tAtj|! dd} tBjCjD| }y$| jE|| |tjkrTtj|= W n6 tFk
r }" z| j9d ||"f W Y d d }"~"X nX | S ) Nr[ r& r c s g | ]}|t |fqS r )rA )r k)r r r r y s zlex..__file__r r __package__rG )r r zCan't build lexerzlex: tokens = %rzlex: literals = %rzlex: states = %rr z
(?P<%s>%s)z(lex: Adding rule %s -> '%s' (state '%s')z#lex: ==== MASTER REGEXS FOLLOW ====z"lex: state '%s' : regex[%d] = '%s'r' zNo t_error rule is definedr z1No error rule is defined for exclusive state '%s'z2No ignore rule is defined for exclusive state '%s'r( z import %sz#Couldn't write lextab module %r. %srI )Gr% r= r rf r dirdictrg r r ru rJ r r r r SyntaxErrorrp r rt r rh r r r r ri r9 r r rV r rQ r; rj r r@ r r r r r+ r, r- r r? extendr0 r) r* r4 r r1 r: r r2 r7 r r r3 r8 rK rL rN re rA rO rP dirnamerd rM )#r rB r optimizer[ r Znowarnr\ ZdebuglogZerrorlogr r ZlexobjZ_itemsZpkgZlinfor{ Zregexsrv Z
regex_listr r r" r r) Zre_textZre_namesr r styper Zsrcfiler Zpkgnamer r )r r lex^ s
$r c C s |sVy&t jd }t|}|j }|j W n* tk
rT t jjd t jj }Y nX | rb| j }nt }|| | rz| j
}nt
}x0| }|sP t jjd|j |j |j
|jf qW d S )Nr( z/Reading from standard input (type EOF to end):
z(%s,%r,%d,%d)
)rf argvrR readclose
IndexErrorstdoutr stdinrt r r r r r )r datar] r Z_inputZ_tokenr r r r runmain s*
r c s fdd}|S )Nc s t drt | _n | _| S )Nr$ )r r r )r )r r r set_regexA s
zTOKEN..set_regexr )r r r )r r TOKEN@ s r )NN)$rS rU rk rf rK r> rO r Z
StringTypeZ UnicodeTyperq AttributeErrorr bytesrl r r r rB r r r! r% r r rZ rm r r r rX VERBOSEr r r Tokenr r r r " sD
F
( 3
@
"