Yfm @ s d Z d d l m Z d Z d d l Z d d l Z d d l Z d d l Z d d l Z d d l Z d d l
m Z d d l m
Z
m Z m Z d d l m Z d d l m Z m Z d d
l m Z d d l m Z d d
d Z Gd d d e Z d d Z d d Z d d Z d d Z e j d d f k rpd d l! Z! e! j" Z# d d Z$ d d Z% n e" Z# e Z$ e Z% d d Z& Gd d! d! e Z' Gd" d# d# e( Z) Gd$ d% d% e Z* Gd& d' d' e) Z+ d S)(zRefactoring framework.
Used as a main program, this can refactor any number of files and/or
recursively descend down directories. Imported as a module, this
provides infrastructure to write your own refactoring tool.
)with_statementz#Guido van Rossum N)chain )drivertokenizetoken) find_root)pytreepygram) btm_utils) btm_matcherTc C s t | g g d g } t j j | j } g } xh t t j | D]Q } | j d rI | j d rI | r | d d } | j | d d qI W| S)zEReturn a sorted list of all available fix names in the given package.*fix_z.py N )
__import__ospathdirname__file__sortedlistdir
startswithendswithappend)Z fixer_pkgZ
remove_prefixZpkgZ fixer_dirZ fix_namesname r 5/opt/alt/python35/lib64/python3.5/lib2to3/refactor.pyget_all_fix_names! s r c @ s e Z d Z d S)
_EveryNodeN)__name__
__module__ __qualname__r r r r r . s r c C s t | t j t j f r: | j d k r0 t | j h St | t j rh | j rb t | j St t | t j r t
} x5 | j D]* } x! | D] } | j t | q Wq W| St d | d S)zf Accepts a pytree Pattern Node and returns a set
of the pattern types which will match first. Nz$Oh no! I don't understand pattern %s)
isinstancer Z NodePatternZ LeafPatterntyper ZNegatedPatternZcontent_get_head_typesZWildcardPatternsetupdate Exception)Zpatrpxr r r r& 2 s
r& c C s t j t } g } x | D] } | j r y t | j } Wn t k
r_ | j | Yq XxU | D] } | | j | qg Wq | j d k r | | j j | q | j | q Wx: t t j
j j t j
j
D] } | | j | q Wt | S)z^ Accepts a list of fixers and returns a dictionary
of head node type --> fixer list. N) collections defaultdictlistpatternr& r r Z _accept_typer r
python_grammarZ
symbol2numbervaluestokensextenddict)Z
fixer_listZ
head_nodesZeveryfixerZheadsZ node_typer r r _get_headnode_dictN s"
r7 c s f d d t d D S)zN
Return the fully qualified names for fixers in the package pkg_name.
c s g | ] } d | q S).r ).0fix_name)pkg_namer r
k s z+get_fixers_from_package..F)r )r; r )r; r get_fixers_from_packageg s r= c C s | S)Nr )objr r r _identityn s r? r c C s | j d d S)Nz
)replace)inputr r r _from_system_newlinesu s rC c C s* t j d k r" | j d t j S| Sd S)Nr@ )r lineseprA )rB r r r _to_system_newlinesw s rE c s d } t j t j | j f d d } t t j t j t j h } t
} yVxO| \ } } | | k r q` q` | t j k r | r Pd } q` | t j k r| d k r| \ } } | t j k s | d k r P| \ } } | t j k s| d k rP| \ } } | t j
k rJ| d k rJ| \ } } x^ | t j k r| j | | \ } } | t j
k s| d k rP| \ } } qMWq` Pq` WWn t k
rYn Xt | S)
NFc s t } | d | d f S)Nr r )next)tok)genr r advance s z(_detect_future_features..advanceTfrom
__future__import(,)r generate_tokensioStringIOreadline frozensetr NEWLINENLCOMMENTr' STRINGNAMEOPadd
StopIteration)sourceZhave_docstringrI ignorefeaturestpvaluer )rH r _detect_future_features sD
ra c @ s e Z d Z d Z d S)
FixerErrorz A fixer could not be loaded.N)r! r" r# __doc__r r r r rb s rb c @ si e Z d Z d d d d i Z d Z d Z d d d d Z d d
Z d d Z d
d Z d d Z
d d Z d d d d Z d d d d Z
d d Z d d d d Z d d Z d d d Z d d Z d! d" Z d d d d# d$ Z d d% d& Z d' Z d( Z d) d* Z d+ d, Z d- d. Z d/ d0 Z d1 d2 Z d3 d4 Z d S)5RefactoringToolprint_functionFwrite_unchanged_filesZFixr Nc C s | | _ | p g | _ | j j | _ | d k rF | j j | | j d rb t j | _ n t j | _ | j j
d | _ g | _ t
j d | _ g | _ d | _ t j | j d t j d | j | _ | j \ | _ | _ g | _ t j | _ g | _ g | _ xz t | j | j D]c } | j rQ| j j! | q/| | j k rs| j j" | q/| | j k r/| j j" | q/Wt# | j | _$ t# | j | _% d S)zInitializer.
Args:
fixer_names: a list of fixers to import
options: a dict with configuration.
explicit: a list of fixers to run even if they are explicit.
Nre rf rd Fconvertlogger)&fixersexplicit_default_optionscopyoptionsr( r
!python_grammar_no_print_statementgrammarr1 getrf errorsloggingZ getLoggerrh fixer_logwroter ZDriverr rg
get_fixers pre_order
post_orderfilesbmZ
BottomMatcherBMZ
bmi_pre_orderZbmi_post_orderr Z
BM_compatibleZ add_fixerr r7 bmi_pre_order_headsbmi_post_order_heads)selfZ fixer_namesrm rj r6 r r r __init__ s<
zRefactoringTool.__init__c C s g } g } x| j D]} t | i i d g } | j d d d } | j | j ru | t | j d } | j d } | j d j d d | D } y t | | } Wn( t
k
r t d | | f Yn X| | j | j
} | j r4| j d
k r4| | j k r4| j d | q | j d | | j d
k rc| j | q | j d k r| j | q t d | j q Wt j d }
| j d |
| j d |
| | f S)a Inspects the options to load the requested patterns and handlers.
Returns:
(pre_order, post_order), where pre_order is the list of fixers that
want a pre-order AST traversal, and post_order is the list that want
post-order traversal.
r
r8 r N_ c S s g | ] } | j q Sr )title)r9 r+ r r r r< s z.RefactoringTool.get_fixers..zCan't find %s.%sTzSkipping optional fixer: %szAdding transformation: %sZpreZpostzIllegal fixer order: %rZ run_orderkey)ri r rsplitr FILE_PREFIXlensplit CLASS_PREFIXjoingetattrAttributeErrorrb rm rs rj log_message log_debugorderr operator
attrgettersort) r} Zpre_order_fixersZpost_order_fixersZ fix_mod_pathmodr: partsZ
class_nameZ fix_classr6 Zkey_funcr r r ru s8 #
zRefactoringTool.get_fixersc O s d S)z Called when an error occurs.Nr )r} msgargskwdsr r r log_error s zRefactoringTool.log_errorc G s$ | r | | } | j j | d S)zHook to log a message.N)rh info)r} r r r r r r s
zRefactoringTool.log_messagec G s$ | r | | } | j j | d S)N)rh debug)r} r r r r r r s
zRefactoringTool.log_debugc C s d S)zTCalled with the old version, new version, and filename of a
refactored file.Nr )r} old_textnew_textfilenameequalr r r print_output! s z RefactoringTool.print_outputc C sP xI | D]A } t j j | r5 | j | | | q | j | | | q Wd S)z)Refactor a list of files and directories.N)r r isdir refactor_dir
refactor_file)r} itemswrite
doctests_onlyZ dir_or_filer r r refactor& s
zRefactoringTool.refactorc
C s t j d } x t j | D] \ } } } | j d | | j | j xb | D]Z } | j d rW t j j | d | k rW t j j | | } | j | | | qW Wd d | D | d d .N)
r extsepwalkr r r r splitextr r )
r} Zdir_namer r Zpy_extdirpathZdirnames filenamesr fullnamer r r r / s
z RefactoringTool.refactor_dirc C s y t | d } Wn< t k
rQ } z | j d | | d SWYd d } ~ Xn Xz t j | j d } Wd | j Xt | d d | } t | j | f SWd QRXd S)zG
Do our best to decode a Python source file correctly.
rbzCan't open %s: %sNr r* encoding)NN)
openOSErrorr r detect_encodingrR close_open_with_encodingrC read)r} r ferrr r r r _read_python_sourceC s z#RefactoringTool._read_python_sourcec C s | j | \ } } | d k r% d S| d 7} | r | j d | | j | | } | j sl | | k r | j | | | | | q| j d | ni | j | | } | j s | r | j r | j t | d d | d | d | n | j d | d S)
zRefactors a file.Nr@ zRefactoring doctests in %szNo doctest changes in %sr r r zNo changes in %sr )r r refactor_docstringrf processed_filerefactor_string was_changedstr)r} r r r rB r outputtreer r r r S s
z RefactoringTool.refactor_filec C s t | } d | k r' t j | j _ zb y | j j | } WnE t k
r } z% | j d | | j j | d SWYd d } ~ Xn XWd | j | j _ X| | _
| j d | | j | | | S)aF Refactor a given input string.
Args:
data: a string holding the code to be refactored.
name: a human-readable name for use in error/log messages.
Returns:
An AST corresponding to the refactored input stream; None if
there were errors during the parse.
re zCan't parse %s: %s: %sNzRefactoring %s)
ra r
rn r ro Z parse_stringr) r __class__r! future_featuresr
refactor_tree)r} datar r^ r r r r r r j s z RefactoringTool.refactor_stringc C s t j j } | ro | j d | j | d } | j sI | | k r_ | j | d | q | j d nS | j | d } | j s | r | j r | j t | d | n
| j d d S)Nz Refactoring doctests in stdinzzNo doctest changes in stdinzNo changes in stdin)
sysstdinr r r rf r r r r )r} r rB r r r r r refactor_stdin s
z RefactoringTool.refactor_stdinc
C s x- t | j | j D] } | j | | q W| j | j | j | j | j | j | j j | j } xt
| j rLx| j j D]} | | k r | | r | | j
d t j j d d | j r | | j
d t j j xGt | | D]5} | | | k r3| | j | y t | Wn t k
rXw Yn X| j rt| | j k rtq | j | } | r | j | | } | d k r | j | x6 | j D]( } | j sg | _ | j j | qW| j j | j } x9 | D]1 } | | k r(g | | <| | j | | q Wq Wq Wq} Wx- t | j | j D] } | j | | qcW| j S)a Refactors a parse tree (modifying the tree in place).
For compatible patterns the bottom matcher module is
used. Otherwise the tree is traversed node-to-node for
matches.
Args:
tree: a pytree.Node instance representing the root of the tree
to be refactored.
name: a human-readable name for this tree.
Returns:
True if the tree was modified, False otherwise.
r reverseTN) r rv rw Z
start_tree traverse_byr{ r| rz ZrunZleavesanyr2 ri r r ZBaseZdepthZkeep_line_orderZ
get_linenor/ remover
ValueErrorZfixers_appliedmatch transformrA r r4 Z finish_treer )
r} r r r6 Z match_setnoderesultsnewZ new_matchesZfxrr r r r sJ
% z RefactoringTool.refactor_treec C s} | s
d Sxl | D]d } x[ | | j D]L } | j | } | r% | j | | } | d k r% | j | | } q% Wq Wd S)a Traverse an AST, applying a set of fixers to each node.
This is a helper method for refactor_tree().
Args:
fixers: a list of fixer instances.
traversal: a generator that yields AST nodes.
Returns:
None
N)r% r r rA )r} ri Z traversalr r6 r r r r r r s
zRefactoringTool.traverse_byc C s | j j | | d k r? | j | d } | d k r? d S| | k } | j | | | | | r | j d | | j s d S| r | j | | | | n | j d | d S)zR
Called when a file has been refactored and there may be changes.
Nr zNo changes to %szNot writing changes to %s)rx r r r r rf
write_file)r} r r r r r r r r r r s z RefactoringTool.processed_filec % C s y t | d d | } Wn< t k
rW } z | j d | | d SWYd d } ~ Xn XzV y | j t | Wn8 t k
r } z | j d | | WYd d } ~ Xn XWd | j X| j d | d | _ d S)zWrites a string to a file.
It first shows a unified diff between the old text and the new text, and
then rewrites the file; the latter is only done if the write option is
set.
wr zCan't create %s: %sNzCan't write %s: %szWrote changes to %sT)r r r r rE r r rt )r} r r r r r r r r r r s * zRefactoringTool.write_filez>>> z... c
C s g } d } d } d } d } x.| j d d D]} | d 7} | j j | j r | d k r | j | j | | | | | } | g } | j | j } | d | } q1 | d k r| j | | j s | | | j j d k r| j | q1 | d k r2| j | j | | | | d } d } | j | q1 W| d k rz| j | j | | | | d j
| S)a Refactors a docstring, looking for doctests.
This returns a modified version of the input string. It looks
for doctests, which start with a ">>>" prompt, and may be
continued with "..." prompts, as long as the "..." is indented
the same as the ">>>".
(Unfortunately we can't use the doctest module's parser,
since, like most parsers, it is not geared towards preserving
the original source.)
Nr keependsTr r@ r )
splitlineslstripr PS1r4 refactor_doctestfindPS2rstripr r )
r} rB r resultblockZ block_linenoindentlinenolineir r r r ( s:
z"RefactoringTool.refactor_docstringc
s^ y j | | } Wn t k
r } zg j j t j rm x' | D] } j d | j d qJ W j d | | | j j
| | SWYd d } ~ Xn X j | | rZt | j
d d } | d | d | | d d } } | d j d s| d d 7< j | j d g } | rZ| f d d
| D 7} | S)
zRefactors one doctest.
A doctest is given as a block of lines, the first of which starts
with ">>>" (possibly indented), while the remaining lines start
with "..." (identically indented).
z
Source: %sr@ z+Can't parse docstring in %s line %s: %s: %sNr Tr r c s! g | ] } j | q Sr )r )r9 r )r r} r r r< m s z4RefactoringTool.refactor_doctest..r r ) parse_blockr) rh Z isEnabledForrr DEBUGr r r r r! r r r r r pop)
r} r r r r r r r r Zclippedr )r r} r r S s$
) z RefactoringTool.refactor_doctestc C s | j r d } n d } | j s4 | j d | n1 | j d | x | j D] } | j | qN W| j r | j d x | j D] } | j | q W| j rt | j d k r | j d n | j d t | j x- | j D]" \ } } } | j | | | q Wd S) Nwerez
need to bezNo files %s modified.zFiles that %s modified:z$Warnings/messages while refactoring:r zThere was 1 error:zThere were %d errors:)rt rx r rs rq r )r} r filemessager r r r r r summarizep s$
zRefactoringTool.summarizec C s1 | j j | j | | | } t | _ | S)zParses a block into a tree.
This is necessary to get correct line number / offset information
in the parser diagnostics and embedded into the parse tree.
)r Z parse_tokens wrap_toksrS r )r} r r r r r r r r s ! zRefactoringTool.parse_blockc c s t j | j | | j } xe | D]] \ } } \ } } \ } }
} | | d 7} | | d 7} | | | | f | |
f | f Vq% Wd S)z;Wraps a tokenize stream to systematically modify start/end.r N)r rO gen_lines__next__) r} r r r r3 r% r` Zline0Zcol0Zline1Zcol1Z line_textr r r r s
(zRefactoringTool.wrap_toksc c s | | j } | | j } | } xo | D]g } | j | rT | t | d Vn4 | | j d k rr d Vn t d | | f | } q' Wx d Vq Wd S)zGenerates lines as expected by tokenize from a list of lines.
This strips the first len(indent + self.PS1) characters off each line.
Nr@ zline=%r, prefix=%rr )r r r r r AssertionError)r} r r Zprefix1Zprefix2prefixr r r r r s
zRefactoringTool.gen_lines) r! r" r# rk r r r~ ru r r r r r r r r r r r r r r r r r r r r r r r r r r rd s: 4 ( O +
rd c @ s e Z d Z d S)MultiprocessingUnsupportedN)r! r" r# r r r r r s r c sa e Z d Z f d d Z d d d f d d Z f d d Z f d d
Z S) MultiprocessRefactoringToolc s/ t t | j | | d | _ d | _ d S)N)superr r~ queue output_lock)r} r kwargs)r r r r~ s z$MultiprocessRefactoringTool.__init__Fr c sU | d k r( t t j | | | Sy d d l Wn t k
rR t Yn X j d k rn t d j _ j _
f d d t | D } z; x | D] } | j q Wt t j | | | Wd j j
x$ t | D] } j j d qWx$ | D] } | j r'| j
q'Wd _ Xd S)Nr r z already doing multiple processesc s% g | ] } j d j q S)target)ZProcess_child)r9 r )multiprocessingr} r r r< s z8MultiprocessRefactoringTool.refactor..)r r r r ImportErrorr r RuntimeErrorZ
JoinableQueueZLockr rangestartr putZis_alive)r} r r r Z
num_processesZ processesr+ r )r )r r} r r s2
z$MultiprocessRefactoringTool.refactorc
so | j j } xY | d k rj | \ } } z t t | j | | Wd | j j X| j j } q Wd S)N)r rp r r r Z task_done)r} Ztaskr r )r r r r s z"MultiprocessRefactoringTool._childc sE | j d k r( | j j | | f n t t | j | | Sd S)N)r r r r r )r} r r )r r r r s z)MultiprocessRefactoringTool.refactor_file)r! r" r# r~ r r r r r )r r r s
r ),rc rK r
__author__r r rr r r- rP itertoolsr Zpgen2r r r Z
fixer_utilr r r r
r Zbur ry r r) r r& r7 r= r? version_infocodecsr r rC rE ra rb objectrd r r r r r r sF
(