3
vh @ s d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl
m Z ddl m
Z
ddlmZmZmZmZ ddlZddlmZmZmZmZmZmZmZm Z m Z m Z m Z m Z m!Z! ddlm"Z" ddl#m$Z$ dd l%m&Z& dd
l'm(Z( dd l)m*Z* dd l+m,Z, dd
l-m.Z. ej/dZ0ej/dej1Z2ej/dZ3ej/dej1j4Z5dj6 Z7ddddgZ8dZ9dZ:e:j;ejd d Z?d d Z@dEd dZAdFd!d"ZBdGd#d$ZCdedfd%dZDdHd&d'ZEd(d) ZFej/d*ej1ZGeFd+d, ZHG d-d. d.eIZJG d/d0 d0eJZKG d1d deZLej/d2jMZNd3d4 ZOd5d6 ZPdId7d8ZQd9d: ZRG d;d< d d>ejTZUejVjWfd?d@ZXeQe9eXZXdAdB ZYdCdD ZZdS )Jz#PyPI and direct package downloading N)wraps)six)urllib http_client configparsermap)
CHECKOUT_DIST Distribution BINARY_DISTnormalize_path SOURCE_DIST Environmentfind_distributions safe_name safe_version to_filename Requirement DEVELOP_DISTEGG_DIST) ssl_support)log)DistutilsError) translate)get_all_headers)unescape)Wheelz^egg=([-A-Za-z0-9_.+!]+)$zhref\s*=\s*['"]?([^'"> ]+)z([^<]+)
\s+\(md5\)z([-+.a-z0-9]{2,}):z .tar.gz .tar.bz2 .tar .zip .tgz PackageIndexdistros_for_urlparse_bdist_wininstinterpret_distro_name z s z(interpret_distro_name..r9 Nr7 )
py_versionrH rV )r? anyrangerT r join)rP rU rM r^ rH rV rA r\ r& r& r' r s
$c c sn t }|j}|dkr>xTtjj|j| D ]}|| |V q&W n,x*| D ]"}||}||krD|| |V qDW dS )zHList unique elements, preserving order. Remember all elements ever seen.N)setaddr Zmoves filterfalse __contains__)iterablekeyseenZseen_addelementkr& r& r' unique_everseen s
rk c s t fdd}|S )zs
Wrap a function returning an iterable such that the resulting iterable
only ever yields unique items.
c s t | |S )N)rk )argskwargs)funcr& r' wrapper s z unique_values..wrapper)r )rn ro r& )rn r'
unique_values s rp z3<([^>]*\srel\s{0,10}=\s{0,10}['"]?([^'" >]+)[^>]*)>c c s xvt j|D ]h}|j \}}tttj|j jd}d|ksFd|kr x,t j|D ] }t
j j | t
|jdV qRW q W xHdD ]@}|j|}|d kr~t j||}|r~t
j j | t
|jdV q~W dS )
zEFind rel="homepage" and rel="download" links in `page`, yielding URLs,Zhomepager8 r7
| Home Page | Download URLN)rr rs r; )RELfinditergroupsrb r strstripr0 r? HREFr r# urljoin
htmldecoderL findsearch)r@ pagerK tagZrelZrelsposr& r& r' find_external_links s "
r c @ s( e Zd ZdZdd Zdd Zdd ZdS ) ContentCheckerzP
A null content checker that defines the interface for checking content
c C s dS )z3
Feed a block of data to the hash.
Nr& )selfblockr& r& r' feed s zContentChecker.feedc C s dS )zC
Check the hash. Return False if validation fails.
Tr& )r r& r& r' is_valid s zContentChecker.is_validc C s dS )zu
Call reporter with information about the checker (hash name)
substituted into the template.
Nr& )r reportertemplater& r& r' report s zContentChecker.reportN)__name__
__module__ __qualname____doc__r r r r& r& r& r' r s r c @ sB e Zd ZejdZdd Zedd Zdd Z dd Z
d
d Z d S )
HashCheckerzK(?Psha1|sha224|sha384|sha256|sha512|md5)=(?P[a-f0-9]+)c C s || _ tj|| _|| _d S )N) hash_namehashlibnewhashexpected)r r r r& r& r' __init__ s zHashChecker.__init__c C s> t jj|d }|st S | jj|}|s0t S | f |j S )z5Construct a (possibly null) ContentChecker from a URLr7 r; )r r# r= r patternr} groupdict)clsr@ rF rK r& r& r' from_url s zHashChecker.from_urlc C s | j j| d S )N)r update)r r r& r& r' r s zHashChecker.feedc C s | j j | jkS )N)r Z hexdigestr )r r& r& r' r ! s zHashChecker.is_validc C s || j }||S )N)r )r r r msgr& r& r' r $ s
zHashChecker.reportN) r r r rZ compiler r classmethodr r r r r& r& r& r' r s r c s< e Zd ZdZdKddZdLd d
ZdMd d ZdNd
dZdd Zdd Z dd Z
dd Z dOddZ dd Z
dP fdd Zd d Zd d Zd!d" Zd#d$ Zd%d& ZdQd'd(ZdRd)d*Zd+d, Zd-Zd.d/ Zd0d1 ZdSd2d3Zd4d5 Zd6d7 Z d8d9 Z d:d; Z dd?Z!d@dA Z"dBdC Z#dDdE Z$dFdG Z%dHdI Z& Z'S )Ur z;A distribution index that scans web pages for download URLshttps://pypi.org/simple/*NTc O s t j| f|| |dd |jd | _i | _i | _i | _tjdj t
t |j | _
g | _|ortjor|prtj }|rtj|| _n
tjj| _d S )Nr6 |)r
r r1 index_url scanned_urls fetched_urls
package_pagesrZ r ra r r rK allowsto_scanr Z is_availableZfind_ca_bundleZ
opener_foropenerr requesturlopen)r r ZhostsZ ca_bundleZ
verify_sslrl kwZuse_sslr& r& r' r , s zPackageIndex.__init__Fc C s || j kr| rdS d| j |< t|s4| j| dS tt|}|r^| j|sRdS | jd| |sr| sr|| jkrtt| j | dS | j|sd| j|< dS | j
d| d| j|< d}| j ||| }|dkrdS d| j|j < d|j
jddj kr|j dS |j }|j }t|tsRt|tjjr4d }n|j
jd
pDd }|j|d }|j x6tj|D ](} tjj |t | j d }
| j |
qfW |j | j!rt"|d
ddkr| j#||}dS )z.)filterrW rD r r itertoolsstarmap
scan_egg_link)r Z search_pathdirsZ egg_linksr& r& r' scan_egg_links s zPackageIndex.scan_egg_linksc
C s t tjj|| }ttd ttj|}W d Q R X t |dkrDd S |\}}x>t
tjj||D ](}tjj|f| |_ t |_
| j| q`W d S )Nr9 )openrW rD ra r r r rw rx rT r rP r rH rc )r rD r Z raw_lineslinesZegg_pathZ
setup_pathrN r& r& r' r s zPackageIndex.scan_egg_linkc
s fdd}xHt j|D ]:}y |tjj|t|jd W q tk
rP Y qX qW ||\}}|rxXt||D ]J}t |\}} |j
dr| r|r|d||f 7 }n
j | j | qrW t
jdd |S dS d S )
z#Process the contents of a PyPI pagec s | j jrtttjj| t jd jd}t|dkrd|d krt |d }t
|d }d j j |j
i | < t|t|fS dS )Nr6 r9 r: r7 r T)NN)r2 r r r r r# r> rT r? r r r
setdefaultr0 r )r rA pkgver)r r& r' scan s " z(PackageIndex.process_index..scanr7 z.pyz
#egg=%s-%sc S s d| j ddd S )Nz%sr7 r! r9 )rL )mr& r& r' s z,PackageIndex.process_index..r N)ry ru r r# rz r{ rL r$ r rG r1 need_version_infoscan_urlPYPI_MD5sub)
r r@ r~ r rK r r new_urlr4 fragr& )r r' r s$
zPackageIndex.process_indexc C s | j d| d S )NzPPage at %s links to .py file(s) without version info; an index scan is required.)scan_all)r r@ r& r& r' r s z PackageIndex.need_version_infoc G s: | j | jkr*|r | j|f| | jd | j| j d S )Nz6Scanning index of all packages (this may take a while))r r r r r )r r rl r& r& r' r s zPackageIndex.scan_allc C s~ | j | j|j d | jj|js:| j | j|j d | jj|jsR| j| x&t| jj|jf D ]}| j | qhW d S )Nr6 ) r r unsafe_namer r rg rQ not_found_in_indexr )r requirementr@ r& r& r'
find_packages s
zPackageIndex.find_packagesc sR | j | j| x,| |j D ] }||kr.|S | jd|| q W tt| j||S )Nz%s does not match %s)prescanr rg r superr obtain)r r Z installerrN ) __class__r& r' r s
zPackageIndex.obtainc C sL |j | jd| |j sH|j tj| td|jjtj j
|f dS )z-
checker is a ContentChecker
z Validating %%s checksum for %sz7%s validation failed for %s; possible download problem?N) r r r r rW unlinkr r r3 rD rU )r checkerrX tfpr& r& r'
check_hash s
zPackageIndex.check_hashc C sT xN|D ]F}| j dks4t| s4|jds4tt|r@| j| q| j j| qW dS )z;Add `urls` to the list that will be prescanned for searchesNzfile:)r r r2 r r r append)r Zurlsr@ r& r& r' add_find_links
s
zPackageIndex.add_find_linksc C s" | j rtt| j| j d| _ dS )z7Scan urls scheduled for prescanning (e.g. --find-links)N)r r r r )r r& r& r' r s zPackageIndex.prescanc C s< | |j r| jd }}n | jd }}|||j | j d S )Nz#Couldn't retrieve index page for %rz3Couldn't find index page for %r (maybe misspelled?))rg r r r r )r r methr r& r& r' r " s
z PackageIndex.not_found_in_indexc C s~ t |tsjt|}|rR| j|jd||}t|\}}|jdrN| j|||}|S tj j
|rb|S t |}t | j
||ddS )a Locate and/or download `spec` to `tmpdir`, returning a local path
`spec` may be a ``Requirement`` object, or a string containing a URL,
an existing local filename, or a project/version requirement spec
(i.e. the string form of a ``Requirement`` object). If it is the URL
of a .py file with an unambiguous ``#egg=name-version`` tag (i.e., one
that escapes ``-`` as ``_`` throughout), a trivial ``setup.py`` is
automatically created alongside the downloaded file.
If `spec` is a ``Requirement`` object or a string containing a
project/version requirement spec, this method returns the location of
a matching distribution (possibly after downloading it to `tmpdir`).
If `spec` is a locally existing file or directory name, it is simply
returned unchanged. If `spec` is a URL, it is downloaded to a subpath
of `tmpdir`, and the local filename is returned. Various errors may be
raised if a problem occurs during downloading.
r7 z.pyrP N)r r r
_download_urlrL rG r1 gen_setuprW rD r r( r fetch_distribution)r r% tmpdirrB foundr4 rF r& r& r' r8 , s
zPackageIndex.downloadc s j d| i d}d
fdd }|rHj j| ||}| r`|dk r`|||}|dkrjdk rzj ||}|dkr| rj| ||}|dkrjdrdpd| nj d| |j|jd S dS ) a| Obtain a distribution suitable for fulfilling `requirement`
`requirement` must be a ``pkg_resources.Requirement`` instance.
If necessary, or if the `force_scan` flag is set, the requirement is
searched for in the (online) package index as well as the locally
installed packages. If a distribution matching `requirement` is found,
the returned distribution's ``location`` is the value you would have
gotten from calling the ``download()`` method with the matching
distribution's URL or filename. If no matching distribution is found,
``None`` is returned.
If the `source` flag is set, only source distributions and source
checkout links will be considered. Unless the `develop_ok` flag is
set, development and system eggs (i.e., those using the ``.egg-info``
format) will be ignored.
zSearching for %sNc s |d kr }x|| j D ]t}|jtkrJ rJ|krjd| d|< q|| ko`|jtkp` }|rj|j}||_tj j
|jr|S qW d S )Nz&Skipping development or system egg: %sr7 ) rg rH r r r r8 rP download_locationrW rD r )ZreqenvrN Ztestloc)
develop_okr skippedsourcer r& r' r| f s z-PackageIndex.fetch_distribution..findz:No local packages or working download links found for %s%sza source distribution of r zBest match: %s)rP )N)r r r r r cloner ) r r r
force_scanr r Z local_indexrN r| r& )r r r r r r' r N s0
z PackageIndex.fetch_distributionc C s" | j ||||}|dk r |jS dS )a3 Obtain a file suitable for fulfilling `requirement`
DEPRECATED; use the ``fetch_distribution()`` method now instead. For
backward compatibility, this routine is identical but returns the
``location`` of the downloaded distribution instead of a distribution
object.
N)r rP )r r r r r rN r& r& r' fetch s zPackageIndex.fetchc
C s t j|}|r*dd t||jdd D p,g }t|dkrtjj|}tjj||krtjj ||}ddl
m } |||st j
|| |}ttjj |dd2} | jd|d j|d jtjj|d f W d Q R X |S |rtd ||f ntd
d S ) Nc S s g | ]}|j r|qS r& )rR )r[ dr& r& r'
s z*PackageIndex.gen_setup..r7 r )samefilezsetup.pywzIfrom setuptools import setup
setup(name=%r, version=%r, py_modules=[%r])
zCan't unambiguously interpret project/version identifier %r; any dashes in the name or version should be escaped using underscores. %rzpCan't process plain .py files without an '#egg=name-version' suffix to enable automatic setup script generation.)rJ rK r rL rT rW rD rU dirnamera Z setuptools.command.easy_installr shutilZcopy2r writerQ rR splitextr )
r rX rF r rK r rU dstr r r& r& r' r s2
zPackageIndex.gen_setupi c C s | j d| d }ztj|}| j|}t|tjjrJtd||j |j
f |j }d}| j }d}d|krt |d} t
tt| }| j||||| t|dZ}
xD|j|} | r|j| |
j| |d7 }| j||||| qP qW | j|||
W d Q R X |S |r|j X d S ) NzDownloading %szCan't download %s: %s %sr r7 zcontent-lengthzContent-Lengthwbr; )r r r r r r r r r r r dl_blocksizer maxr int
reporthookr r r r r r ) r r@ rX fpr r blocknumZbssizeZsizesr r r& r& r' _download_to s:
zPackageIndex._download_toc C s d S )Nr& )r r@ rX r Zblksizer r& r& r' r s zPackageIndex.reporthookc C s |j drt|S y t|| jS ttjfk
r } z>djdd |jD }|r^| j || nt
d||f W Y d d }~X n t j j
k
r } z|S d }~X n t j jk
r } z,|r| j ||j nt
d||jf W Y d d }~X n tjk
r8 } z.|r| j ||j nt
d||jf W Y d d }~X nP tjtj fk
r } z*|rf| j || nt
d||f W Y d d }~X nX d S )Nzfile: c S s g | ] }t |qS r& )rw )r[ argr& r& r' r s z)PackageIndex.open_url..z%s %szDownload error for %s: %sz;%s returned a bad status line. The server might be down, %s)r2
local_openopen_with_authr r$ r
InvalidURLra rl r r r r r ZURLErrorreasonZ
BadStatusLinelineZ
HTTPExceptionsocket)r r@ Zwarningvr r& r& r' r s6
" zPackageIndex.open_urlc C s t |\}}|r4x&d|kr0|jddjdd}qW nd}|jdrN|d d }tjj||}|jt|sztdj |d |d
ks|jd r| j
||S |d ks|jd
r| j ||S |jdr| j ||S |dkrt
jjt
jj|d S | j|d | j||S d S )Nz...\_Z__downloaded__z.egg.zipr, zInvalid filename {filename})rX svnzsvn+gitzgit+zhg+r r9 Tr/ )rG replacer1 rW rD ra r2 rw r$ format
_download_svn
_download_git _download_hgr r url2pathnamer# r= r _attempt_download)r rB r@ r r3 rF rX r& r& r' r s(
zPackageIndex._download_urlc C s | j |d d S )NT)r )r r@ r& r& r' r = s zPackageIndex.scan_urlc C s6 | j ||}d|jddj kr.| j|||S |S d S )Nr z content-typer )r r r0 _download_html)r r@ rX r r& r& r' r) @ s z PackageIndex._attempt_downloadc C sl t |}x@|D ]8}|j rtjd|rD|j tj| | j||S P qW |j tj| td| d S )Nz ([^- ]+ - )?Revision \d+:z Unexpected HTML page found at ) r rx rZ r} r rW r r% r )r r@ r rX r r r& r& r' r* G s
zPackageIndex._download_htmlc C s |j ddd }g }|j jdrd|krtjj|\}}}}}} | r|jdrd|dd kr|dd j dd\}}tjj|\}
} |
rd |
kr|
j d d\} }
d
| d |
g}n
d
|
g}| }|||||| f}tjj|}| jd || t j
d
dg| d||g |S )Nr: r7 r zsvn:@z//r6 r9 :z --username=z --password=z'Doing subversion checkout from %s to %sr! checkoutz-q) r? r0 r2 r r# r= splituser
urlunparser
subprocess
check_call)r r@ rX ZcredsrB netlocrD r\ qr authhostuserZpwrA r& r& r' r% V s$
zPackageIndex._download_svnc C sp t jj| \}}}}}|jddd }|jddd }d }d|krR|jdd\}}t jj||||df} | |fS )N+r7 r: r r+ r r; )r r# Zurlsplitr? rsplitZ
urlunsplit)r@
pop_prefixrB r2 rD rE r revr& r& r' _vcs_split_rev_from_urlk s z$PackageIndex._vcs_split_rev_from_urlc C sr |j ddd }| j|dd\}}| jd|| tjddd ||g |d k rn| jd
| tjdd |d d |g |S )
Nr: r7 r T)r9 z Doing git clone from %s to %sr" r z--quietzChecking out %sz-Cr- )r? r; r r0 r1 )r r@ rX r: r& r& r' r& } s zPackageIndex._download_gitc C sv |j ddd }| j|dd\}}| jd|| tjddd ||g |d k rr| jd
| tjdd |d d
d|dg |S )Nr: r7 r T)r9 z Doing hg clone from %s to %sZhgr z--quietzUpdating to %sz--cwdZupz-Cz-rz-q)r? r; r r0 r1 )r r@ rX r: r& r& r' r' s zPackageIndex._download_hgc G s t j|f| d S )N)r r )r r rl r& r& r' r s zPackageIndex.debugc G s t j|f| d S )N)r r )r r rl r& r& r' r s zPackageIndex.infoc G s t j|f| d S )N)r r )r r rl r& r& r' r s zPackageIndex.warnr )r r< NT)F)F)F)N)N)FFFN)FF)N)F)(r r r r r r r r r r r r r r r r r r r r8 r r r r
r r r r r r) r* r% staticmethodr; r& r' r r r
__classcell__r& r& )r r' r ) sL
3
+
#
J
)$
#!
z!&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?c C s | j d}t|S )Nr7 )rL r )rK Zwhatr& r& r'
decode_entity s
r? c C s
t t| S )z'Decode HTML entities in the given text.)
entity_subr? )textr& r& r' r{ s r{ c s fdd}|S )Nc s fdd}|S )Nc s. t j }t j z
| |S t j| X d S )N)r ZgetdefaulttimeoutZsetdefaulttimeout)rl rm Z old_timeout)rn timeoutr& r' _socket_timeout s
z@socket_timeout.<locals>._socket_timeout.<locals>._socket_timeoutr& )rn rC )rB )rn r' rC s z'socket_timeout.<locals>._socket_timeoutr& )rB rC r& )rB r' socket_timeout s rD c C s2 t jj| }|j }tj|}|j }|jddS )aq
A function compatible with Python 2.3-3.3 that will encode
auth from a URL suitable for an HTTP header.
>>> str(_encode_auth('username%3Apassword'))
'dXNlcm5hbWU6cGFzc3dvcmQ='
Long auth strings should not cause a newline to be inserted.
>>> long_auth = 'username:' + 'password'*10
>>> chr(10) in str(_encode_auth(long_auth))
False
r )r r# r> encodebase64Z encodestringr r# )r4 Zauth_sZ
auth_bytesZ
encoded_bytesZencodedr& r& r' _encode_auth s
rH c @ s( e Zd ZdZdd Zdd Zdd ZdS )
Credentialz:
A username/password pair. Use like a namedtuple.
c C s || _ || _d S )N)usernamepassword)r rJ rK r& r& r' r s zCredential.__init__c c s | j V | jV d S )N)rJ rK )r r& r& r' __iter__ s zCredential.__iter__c C s dt | S )Nz%(username)s:%(password)s)vars)r r& r& r' __str__ s zCredential.__str__N)r r r r r rL rN r& r& r& r' rI s rI c @ s0 e Zd Zdd Zedd Zdd Zdd Zd S )
PyPIConfigc C sP t jdddgd}tjj| | tjjtjjdd}tjj |rL| j
| dS )z%
Load from ~/.pypirc
rJ rK
repositoryr ~z.pypircN) dictfromkeysr RawConfigParserr rW rD ra
expanduserr r )r ZdefaultsZrcr& r& r' r s
zPyPIConfig.__init__c s& fdd j D }tt j|S )Nc s g | ]} j |d j r|qS )rP )r rx )r[ section)r r& r' r s z2PyPIConfig.creds_by_repository.<locals>.<listcomp>)ZsectionsrR r _get_repo_cred)r Zsections_with_repositoriesr& )r r' creds_by_repository s z PyPIConfig.creds_by_repositoryc C s6 | j |dj }|t| j |dj | j |dj fS )NrP rJ rK )r rx rI )r rV Zrepor& r& r' rW s zPyPIConfig._get_repo_credc C s* x$| j j D ]\}}|j|r |S q W dS )z
If the URL indicated appears to be a repository defined in this
config, return the credential for that repository.
N)rX itemsr2 )r r@ rP credr& r& r' find_credential s
zPyPIConfig.find_credentialN)r r r r propertyrX rW r[ r& r& r& r' rO s rO c C s: t jj| \}}}}}}|jdr,tjd|d
krFt jj|\}} nd}|s~t j| }
|
r~t |
}|
j
| f} t j d | |rdt
| }|| ||||f} t jj| }
t jj|
}|jd| n t jj| }|jd t ||}|r6t jj|j\}}}}}}||kr6|| kr6||||||f} t jj| |_|S ) z4Open a urllib2 request, handling HTTP authenticationr, znonnumeric port: ''httphttpsN*Authenticating as %s for %s (from .pypirc)zBasic Z
Authorizationz
User-Agent)r] r^ )r_ )r r# r= r1 r r r. rO r[ rw rJ r r rH r/ r ZRequestZ
add_header
user_agentr@ )r@ r rB r2 rD ZparamsrE r r4 r5 rZ r rA r r r s2Zh2Zpath2Zparam2Zquery2Zfrag2r& r& r' r
s6
r c C s | S )Nr& )r@ r& r& r'
fix_sf_url> s rb c C s t jj| \}}}}}}t jj|}tjj|r<t jj| S |j drtjj
|rg }xtj |D ]b} tjj || }
| dkrt
|
d} | j } W dQ R X P ntjj
|
r| d7 } |jdj| d qbW d}
|
j| dj |d } d\}}n
d\}}} ddi}tj| }t jj| ||||S )z7Read a local path, with special support for directoriesr6 z
index.htmlrNz<a href="{name}">{name}</a>)r3 zB<html><head><title>{url}{files} |