i fc @ s d Z d Z d Z d d l m Z d d l m Z m Z d d l Z d d l
m Z d d l m
Z
d d l Z d d l Z d d l Te j d
e j Z e j d e j Z d d l Z e j d d
d d d d d g Z [ e Z d e e d9 6e? d: 6e@ d; 6eA d< 6eB d= 6eC d> 6eD d? 6eE d@ 6ZF GdA d d e jG d dB ZH dC dD ZI dE dF ZJ dG dH ZK dI ZL dJ ZM eL eJ dK eL eK eM ZN dL ZO dM ZP dN ZQ dO ZR dP ZS eI eP eQ eR eS ZT dQ ZU eI dR dS eK eU ZV dT eU ZW eI eV eW ZX eI dU eX dV ZY eI eY eX eT ZZ dW Z[ dX Z\ dY Z] dZ Z^ d[ Z_ eI e[ d\ e[ d] Z` eI e[ d^ e[ d_ Za eI d` da db d+ dc dd de d. Zb df Zc eI dg dh di Zd eI eb ec ed Ze eI eZ ee ea eO Zf eN ef Zg eI e[ dj eI dk dK e[ dl eI dm dK Zh eI dn eM e` Zi eL eI ei eZ ee eh eO Zj do dp Zk i( e\ dk 6e] dm 6e^ d\ 6e_ d] 6e^ dq 6e_ dr 6e^ ds 6e_ dt 6e^ du 6e_ dv 6e^ dw 6e_ dx 6e^ dy 6e_ dz 6e^ d{ 6e_ d| 6e^ d} 6e_ d~ 6e^ d 6e_ d 6e^ d 6e_ d 6e^ d 6e_ d 6e^ d 6e_ d 6e^ d 6e_ d 6e^ d 6e_ d 6e^ du 6e_ dv 6e^ d 6e_ d 6d d 6d d 6d d 6d d 6d d 6d d 6Zl i Zm x d D] Zn en em en zpGvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro, Raymond Hettinger, Trent Nelson, Michael Foord )open)lookupBOM_UTF8N)
TextIOWrapper)chain)*z%^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)s ^[ \t\f]*(?:[#\r\n]|$)COMMENTtokenizedetect_encodingNL
untokenizeENCODING TokenInfo ()[]:,;+-r /|&<>=.%{}z==z!=z<=z>=~^z<>z**z+=z-=z*=z/=z%=z&=z|=z^=z<<=z>>=z**=z//z//=@c @ s. e Z d Z d d Z e d d Z d S)r c C s. d | j t | j f } d | j d | S)Nz%d (%s)z8TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r)type)r( tok_name_replace)selfannotated_type r- -/opt/alt/python34/lib64/python3.4/tokenize.py__repr__b s zTokenInfo.__repr__c C s4 | j t k r) | j t k r) t | j S| j Sd S)N)r( OPstringEXACT_TOKEN_TYPES)r+ r- r- r.
exact_typeg s zTokenInfo.exact_typeN)__name__
__module__ __qualname__r/ propertyr3 r- r- r- r. r a s ztype string start end linec G s d d j | d S)Nr r r )join)choicesr- r- r. groupn s r: c G s t | d S)Nr )r: )r9 r- r- r. anyo s r; c G s t | d S)N?)r: )r9 r- r- r. maybep s r= z[ \f\t]*z #[^\r\n]*z\\\r?\nz\w+z0[xX][0-9a-fA-F]+z
0[bB][01]+z 0[oO][0-7]+z(?:0+|[1-9][0-9]*)z[eE][-+]?[0-9]+z[0-9]+\.[0-9]*z\.[0-9]+z[0-9]+z
[0-9]+[jJ]z[jJ]z (?:[bB][rR]?|[rR][bB]?|[uU])?z[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""'''"""z '[^\n'\\]*(?:\\.[^\n'\\]*)*'z "[^\n"\\]*(?:\\.[^\n"\\]*)*"z\*\*=?z>>=?z<<=?z//=?z->z[+\-*/%&|^=<>]=?z[][(){}]z\r?\nz\.\.\.z[:;.,@]z'[^\n'\\]*(?:\\.[^\n'\\]*)*'z"[^\n"\\]*(?:\\.[^\n"\\]*)*"z
\\\r?\n|\Zc C s t j | t j S)N)recompileUNICODE)exprr- r- r. _compile s rF r'''r"""b'''b"""R'''R"""B'''B"""br'''br"""bR'''bR"""Br'''Br"""BR'''BR"""rb'''rb"""Rb'''Rb"""rB'''rB"""RB'''RB"""u'''u"""U'''U"""rRbBuUr'r"R'R"b'b"B'B"br'br"Br'Br"bR'bR"BR'BR"rb'rb"rB'rB"Rb'Rb"RB'RB"u'u"U'U" c @ s e Z d Z d S)
TokenErrorN)r4 r5 r6 r- r- r- r. r s r c @ s e Z d Z d S)StopTokenizingN)r4 r5 r6 r- r- r- r. r s r c @ s@ e Z d Z d d Z d d Z d d Z d d Z d S)
Untokenizerc C s( g | _ d | _ d | _ d | _ d S)Nr r )tokensprev_rowprev_colencoding)r+ r- r- r. __init__ s zUntokenizer.__init__c C s | \ } } | | j k s9 | | j k r` | | j k r` t d j | | | j | j n | | j } | r | j j d | d | _ n | | j } | r | j j d | n d S)Nz+start ({},{}) precedes previous end ({},{})z\
r )r r
ValueErrorformatr append)r+ startrowcol
row_offset
col_offsetr- r- r. add_whitespace s -
zUntokenizer.add_whitespacec C s t | } g } d } x| D]} t | d k rK | j | | Pn | \ } } } } }
| t k r{ | | _ q n | t k r Pn | t k r | j | q n | t k r | j | \ | _
| _ q nr | t t
f k r d } nW | rJ| rJ| d } | d t | k rA| j j | t | | _ n d } n | j | | j j | | \ | _
| _ | t t
f k r | j
d 7_
d | _ q q Wd j | j S)NFr Tr r )iterlencompatr
r ENDMARKERINDENTr DEDENTpopr r NEWLINEr r r r8 ) r+ iterableitindents startlinettok_typetokenr endlineindentr- r- r. r sF
zUntokenizer.untokenizec
C sP g } | j j } | d t t f k } d } xt | g | D]} | d d \ } } | t k rx | | _ qA n | t t f k r | d 7} n | t k r | r d | } n d } n d } | t
k r | j | qA nZ | t k r | j qA n> | t t f k rd } n# | r>| r>| | d d } n | | qA Wd S)Nr Fr r Tr r )
r r r r r r
r NAMENUMBERSTRINGr r r )
r+ r r r toks_appendr
prevstringtoktoknumtokvalr- r- r. r s8
zUntokenizer.compatN)r4 r5 r6 r r r r r- r- r- r. r s
%r c C s@ t } | j | } | j d k r< | j | j } n | S)a Transform tokens back into Python source code.
It returns a bytes object, encoded using the ENCODING
token, which is the first token sequence output by tokenize.
Each element returned by the iterable must be a token sequence
with at least two elements, a token number and token value. If
only two tokens are passed, the resulting output is poor.
Round-trip invariant for full input:
Untokenized source will match input source exactly
Round-trip invariant for limited intput:
# Output bytes will tokenize the back to the input
t1 = [tok[:2] for tok in tokenize(f.readline)]
newcode = untokenize(t1)
readline = BytesIO(newcode).readline
t2 = [tok[:2] for tok in tokenize(readline)]
assert t1 == t2
N)r r r encode)r utoutr- r- r. r ? s
c C sd | d d j j d d } | d k s= | j d rA d S| d
k s\ | j d r` d S| S)z(Imitates get_normal_name in tokenizer.c.N _r zutf-8zutf-8-latin-1
iso-8859-1 iso-latin-1latin-1- iso-8859-1- iso-latin-1-)r r r )r r r )lowerreplace
startswith)orig_encencr- r- r. _get_normal_nameZ s " r c s4 y j j Wn t k
r* d Yn Xd d } d } f d d } f d d } | } | j t r d | d d } d
} n | s | g f S| | } | r | | g f St j | s | | g f S| } | s| | g f S| | } | r$| | | g f S| | | g f S) a
The detect_encoding() function is used to detect the encoding that should
be used to decode a Python source file. It requires one argument, readline,
in the same way as the tokenize() generator.
It will call readline a maximum of twice, and return the encoding used
(as a string) and a list of any lines (left as bytes) it has read in.
It detects the encoding from the presence of a utf-8 bom or an encoding
cookie as specified in pep-0263. If both a bom and a cookie are present,
but disagree, a SyntaxError will be raised. If the encoding cookie is an
invalid charset, raise a SyntaxError. Note that if a utf-8 bom is found,
'utf-8-sig' is returned.
If no encoding is specified, then the default of 'utf-8' will be returned.
NFzutf-8c s( y SWn t k
r# d SYn Xd S)N )
StopIterationr- )readliner- r. read_or_stop} s
z%detect_encoding..read_or_stopc s; y | j d } WnE t k
rZ d } d k rJ d j | } n t | Yn Xt j | } | st d St | j d } y t | } WnI t k
r d k r d | } n d j | } t | Yn X r7| d k r* d k r d } n d j } t | n | d 7} n | S)
Nzutf-8z'invalid or missing encoding declarationz {} for {!r}r zunknown encoding: z unknown encoding for {!r}: {}zencoding problem: utf-8z encoding problem for {!r}: utf-8z-sig)
decodeUnicodeDecodeErrorr SyntaxError cookie_rematchr r: r LookupError)r line_stringmsgr r codec) bom_foundfilenamer- r. find_cookie s6
z$detect_encoding..find_cookieTr z utf-8-sig)__self__nameAttributeErrorr r blank_rer )r r defaultr r firstsecondr- )r r r r. r
e s8
&
c C ss t | d } yH t | j \ } } | j d t | | d d } d | _ | SWn | j Yn Xd S)zXOpen a file in read only mode using the encoding detected by
detect_encoding().
rbr line_bufferingTrc N)
_builtin_openr
r seekr modeclose)r bufferr linestextr- r- r. r s
r c C s_ d d l m } m } t | \ } } t | d } | d } t | | | | j | S)a
The tokenize() generator requires one argment, readline, which
must be a callable object which provides the same interface as the
readline() method of built-in file objects. Each call to the function
should return one line of input as bytes. Alternately, readline
can be a callable function terminating with StopIteration:
readline = open(myfile, 'rb').__next__ # Example of alternate readline
The generator produces 5-tuples with these members: the token type; the
token string; a 2-tuple (srow, scol) of ints specifying the row and
column where the token begins in the source; a 2-tuple (erow, ecol) of
ints specifying the row and column where the token ends in the source;
and the line on which the token was found. The line passed is the
logical line; continuation lines are included.
The first token sequence will always be an ENCODING token
which tells you which encoding was used to decode the bytes stream.
r )r repeatr ) itertoolsr r r
r _tokenize__next__)r r r r consumedrl_genemptyr- r- r. r s
c c s6 d } } } d } d \ } } d } d g } | d k rj | d k rP d } n t t | d d d Vn xdy
| }
Wn t k
r d }
Yn X| d k r |
j | }
n | d 7} d t |
} } | r|
s t d |
n | j |
} | rZ| j d } } t t | |
d | |
| | f | |
Vd \ } } d } ql| r|
d d d
k r|
d! d d k rt t | |
|
| t |
f | Vd } d } qm ql| |
} | |
} qm n| d k rH| rH|
sPn d } xv | | k rz|
| d
k r.| d 7} n? |
| d k rS| t
d t
} n |
| d k rld } n P| d 7} qW| | k rPn |
| d k r|
| d k r7|
| d j d } | t | } t t | | | f | | t | f |
Vt t
|
| d | | f | t |
f |
Vqm t t
t f |
| d k |
| d | | f | t |
f |
Vqm n | | d" k r| j | t t |
d | | d f | | f |
Vn x | | d# k rD| | k rt d d | | |
f n | d d$ } t t d | | f | | f |
VqWn$ |
sft d | d f n d } x^| | k rt t j |
| } | r| j d \ } } | | f | | f | } } } | | k rqon |
| | |
| } } | | k s+| d k rE| d k rE| d k rEt t | | | |
Vq| d k r}t | d k rft
n t | | | |
Vq| d k r| j d st t t | | | |
Vq| t k rRt t | } | j |
| } | r,| j d } |
| | } t t | | | | f |
Vq| | f }
|
| d } |
} Pq| t k s| d d t k s| d d t k r
| d% d k r| | f }
t t | pt | d pt | d } |
| d d } } |
} Pqt t | | | |
Vq| j r3t t | | | |
Vq| d k rHd } q| d k ra| d 7} n | d k rz| d 8} n t t | | | |
Vqot t |
| | | f | | d f |
V| d 7} qoWqm Wx; | d d D]) } t t d | d f | d f d VqWt t d | d f | d f d Vd S)&Nr
0123456789r z utf-8-sigzutf-8r r zEOF in multi-line stringr z\
r z\
r z#
#z
z3unindent does not match any outer indentation levelz
zEOF in multi-line statementr! z...
\z([{z)]})r r )r r )r r )r r r r r r ) r r
r r r r r r r
ERRORTOKENtabsizerstripr r r r IndentationErrorr rF PseudoTokenspanr r endswithAssertionError
triple_quotedendpats
single_quoted isidentifierr r0 r )r r lnumparenlev continuednumcharscontstrneedcontcontliner r posmaxstrstartendprogendmatchr column
comment_tokennl_pos pseudomatchr sposeposr initialr r- r- r. r s
2
$#'
0 * $
'r c C s
t | d S)N)r )r r- r- r. generate_tokens s r c
s d d l } d d d d f d d } | j d d } | j d d d
d d d
d d | j d d d d d d d d | j } y | j r | j } t | d } t t | j } Wd QXn d } t t
j j d } x^ | D]V } | j } | j
r | j } n d | j | j } t d | t | | j f q WWnbt k
r}
z? |
j d d d \ } } | |
j d | | | f WYd d }
~
Xnt k
r }
z5 |
j d \ } } | |
j d | | | f WYd d }
~
Xn t k
r:}
z | |
| WYd d }
~
Xn t k
rh}
z | |
WYd d }
~
XnQ t k
rt d Yn6 t k
r}
z d |
WYd d }
~
Xn Xd S) Nr c S s t | d t j d S)Nfile)printsysstderr)messager- r- r. perror s zmain..perrorc sg | r+ | f | | f } d | n+ | rH d | | f n d | t j d d S)Nz%s:%d:%d: error: %sz
%s: error: %sz error: %sr )r exit)r r locationargs)r r- r. error s zmain..errorprogzpython -m tokenizedestr nargsr< metavarz filename.pyhelpz'the file to tokenize; defaults to stdinz-ez--exactexactaction
store_truez(display token names using the exact typer zz %d,%d-%d,%d:z%-20s%-15s%-15rr r z interrupted
zunexpected error: %s)argparseArgumentParser add_argument
parse_argsr r listr r r r stdinr( r# r3 r r r r) r1 r r r r OSErrorKeyboardInterrupt Exception)
r& r parserr r fr r
token_type token_rangeerrr r r- )r r. main sN
//
r4 __main__) r> r? rG rH rK rL rI rJ rM rN rO rP rS rT rQ rR rU rV rW rX r[ r\ rY rZ r] r^ r_ r` ra rb ) r@ rA ri rj rk rl rm rn ro rp rq rr rs rt ru rv rw rx ry rz r{ r| r} r~ r r r r r r )}__doc__
__author__ __credits__builtinsr r codecsr r collectionsior r r rB r r rC ASCIIr r __all__N_TOKENSr r) r r
LPARRPARLSQBRSQBCOLONCOMMASEMIPLUSMINUSSTARSLASHVBARAMPERLESSGREATEREQUALDOTPERCENTLBRACERBRACEEQEQUALNOTEQUAL LESSEQUAL GREATEREQUALTILDE
CIRCUMFLEX LEFTSHIFT
RIGHTSHIFT
DOUBLESTAR PLUSEQUALMINEQUAL STAREQUAL
SLASHEQUAL PERCENTEQUAL
AMPEREQUAL VBAREQUALCIRCUMFLEXEQUALLEFTSHIFTEQUALRIGHTSHIFTEQUALDOUBLESTAREQUAL DOUBLESLASHDOUBLESLASHEQUALATr2
namedtupler r: r; r=
WhitespaceCommentIgnoreName Hexnumber Binnumber Octnumber Decnumber IntnumberExponent
PointfloatExpfloat Floatnumber
ImagnumberNumber StringPrefixSingleDoubleSingle3Double3TripleStringOperatorBracketSpecialFunny
PlainTokenTokenContStr PseudoExtrasr rF r r r r r r. r r r r r r
r r r r4 r4 r- r- r- r. sF
"
_ ] <