o
XaiΛ @ s d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl
Z
ddl Z ddl Z ddl
ZddlZddlZddlZddlmZ ddlZddlmZmZmZmZmZm Z m Z m Z m Z m Z m!Z!m"Z"m#Z# ddl$m%Z% ddl&m'Z' ddl(m)Z) ddl*m+Z+ dd l,m-Z- e.d
Z/e.d ej0Z1e.d Z2e.d
ej0j3Z4d5 Z6g dZ7dZ8dZ9e9j:dj:ej; edZdd Z?dBddZ@dBd d ZAdBd d ZBdedfd d!ZCd"d# ZDe.d$ej0ZEeDd%d& ZFG d'd( d(ZGG d)d* d*eGZHG d+d, d,e ZIe.d-jJZKd.d/ ZLd0d1 ZMdCd2d3ZNd4d5 ZOG d6d7 d7ZPG d8d9 d9e jQZRejSjTfd:d;ZUdd? ZWd@dA ZXdS )Dz#PyPI and direct package downloading Nwraps)
CHECKOUT_DIST Distribution BINARY_DISTnormalize_path SOURCE_DIST Environmentfind_distributions safe_name safe_version to_filename Requirement DEVELOP_DISTEGG_DIST)log)DistutilsError) translate)Wheelunique_everseenz^egg=([-A-Za-z0-9_.+!]+)$zhref\s*=\s*['"]?([^'"> ]+)z([^<]+)\n\s+\(md5\)z([-+.a-z0-9]{2,}):z .tar.gz .tar.bz2 .tar .zip .tgz) PackageIndexdistros_for_urlparse_bdist_wininstinterpret_distro_name z s z(interpret_distro_name.. Nr3 )
py_versionrA rS )r7 anyrangerP r join)rJ rQ rF r] rA rS r9 rZ r! r! r" r s
$
r c s t fdd}|S )zs
Wrap a function returning an iterable such that the resulting iterable
only ever yields unique items.
c s t | i |S Nr )argskwargsfuncr! r" wrapper s z unique_values..wrapperr )re rf r! rd r"
unique_values s rg z(<([^>]*\srel\s*=\s*['"]?([^'">]+)[^>]*)>c c s t |D ]2}| \}}tttj| d}d|v s#d|v r8t |D ]}t
j | t
|dV q(qdD ] }||}|dkr[t ||}|r[t
j | t
|dV q;dS )zEFind rel="homepage" and rel="download" links in `page`, yielding URLs,Zhomepager0 r3 )z
| Home Pagez | Download URLr/ N)RELfinditergroupssetmapstrstripr' r7 HREFr4 r urljoin
htmldecoderE findsearch)r8 pagerD tagrelZrelsposr! r! r" find_external_links s
ry c @ ( e Zd ZdZdd Zdd Zdd ZdS ) ContentCheckerzP
A null content checker that defines the interface for checking content
c C dS )z3
Feed a block of data to the hash.
Nr! selfblockr! r! r" feed zContentChecker.feedc C r| )zC
Check the hash. Return False if validation fails.
Tr! r~ r! r! r" is_valid r zContentChecker.is_validc C r| )zu
Call reporter with information about the checker (hash name)
substituted into the template.
Nr! )r~ reportertemplater! r! r" report s zContentChecker.reportN)__name__
__module__ __qualname____doc__r r r r! r! r! r" r{ s
r{ c @ sB e Zd ZedZdd Zedd Zdd Z dd Z
d
d Z d S )
HashCheckerzK(?Psha1|sha224|sha384|sha256|sha512|md5)=(?P[a-f0-9]+)c C s || _ t|| _|| _d S ra ) hash_namehashlibnewhashexpected)r~ r r r! r! r" __init__ s
zHashChecker.__init__c C sB t j|d }|s
t S | j|}|st S | di | S )z5Construct a (possibly null) ContentChecker from a URLr/ Nr! )r4 r r5 r{ patternrt groupdict)clsr8 r? rD r! r! r" from_url s zHashChecker.from_urlc C s | j | d S ra )r updater} r! r! r" r zHashChecker.feedc C s | j | jkS ra )r hexdigestr r r! r! r" r r zHashChecker.is_validc C s || j }||S ra )r )r~ r r msgr! r! r" r s
zHashChecker.reportN) r r r rX compiler r classmethodr r r r r! r! r! r" r s
r c sL e Zd ZdZ dLddZdMd d
ZdMd d ZdMd
dZdd Zdd Z dd Z
dd Z dd Z dNddZ
dd ZdN fd d Zd d Zd!d" Zd#d$ Zd%d& Zd'd( Z dOd)d*ZdPd+d,Zd-d. Zd/Zd0d1 Zd2d3 ZdNd4d5Zd6d7 Z d8d9 Z d:d; Z dd? Z e!dMd@dAZ"dBdC Z#dDdE Z$dFdG Z%dHdI Z&dJdK Z' Z(S )Qr z;A distribution index that scans web pages for download URLshttps://pypi.org/simple/*NTc O sr t j| g|R i | |dd |d | _i | _i | _i | _td t
t |j | _
g | _tjj| _d S )Nr. |)r r r( index_url scanned_urls fetched_urls
package_pagesrX r r` rm r rD allowsto_scanr4 requesturlopenopener)r~ r hostsZ ca_bundleZ
verify_sslrb kwr! r! r" r s zPackageIndex.__init__Fc C s || j v r |s dS d| j |< t|s| | dS tt|}|r.| |s(dS | d| |s7|r7|| jv rAtt| j | dS | |sMd| j|< dS |
d| d| j|< d}| ||| }|du rhdS t |t
jjr||jdkr||
d|j d| j|j< d|jd d
vr| dS |j}| }t |tst |t
jjrd }n|jd pd }||d
}| t |D ]} t
j |t | d}
| !|
q|"| j#rt$|dddkr| %||}dS dS dS )z.)filterrU r< r r itertoolsstarmap
scan_egg_link)r~ search_pathdirsZ egg_linksr! r! r" scan_egg_links| s
zPackageIndex.scan_egg_linksc C s t tj||}ttd ttj|}W d n1 s w Y t |dkr,d S |\}}t
tj||D ]}tjj|g|R |_ t |_
| | q9d S )Nr\ )openrU r< r` r r rm rn ro rP r
rJ r rA r )r~ r< r Z raw_lineslinesZegg_pathZ
setup_pathrG r! r! r" r s zPackageIndex.scan_egg_linkc C s d}| | js
|S tttjj|t| jd d}t|dks)d|d v r+|S t |d }t
|d }d| j |
i |< t|t|fS )N)NNr. r\ r2 r3 r T)r) r r rm r4 r r6 rP r7 r r r
setdefaultr' r
)r~ r ZNO_MATCH_SENTINELr9 pkgverr! r! r" _scan s zPackageIndex._scanc
C s t |D ] }z| tj|t|d W q ty! Y qw | |\}}|s-dS t ||D ]$}t
|\}}| drQ|sQ|rL|d||f 7 }n| | |
| q2tdd |S )z#Process the contents of a PyPI pager3 r .pyz
#egg=%s-%sc S s d| ddd S )Nz%sr3 r\ )rE )mr! r! r" s z,PackageIndex.process_index..)rp rj r r4 r rq rr rE r ry r@ r( need_version_infoscan_urlPYPI_MD5sub) r~ r8 ru rD r r new_urlr+ fragr! r! r" r s& "
zPackageIndex.process_indexc C s | d| d S )NzPPage at %s links to .py file(s) without version info; an index scan is required.)scan_allr~ r8 r! r! r" r s z PackageIndex.need_version_infoc G s< | j | jvr|r| j|g|R | d | | j d S )Nz6Scanning index of all packages (this may take a while))r r r r r r~ r rb r! r! r" r s zPackageIndex.scan_allc C sz | | j|j d | j|js | | j|j d | j|js)| | t| j|jdD ]}| | q3d S )Nr. r! ) r r unsafe_namer r keyrK not_found_in_indexr )r~ requirementr8 r! r! r"
find_packages s
zPackageIndex.find_packagesc sR | | | | |j D ]}||v r| S | d|| qtt| ||S )Nz%s does not match %s)prescanr r r superr obtain)r~ r installerrG __class__r! r" r s
zPackageIndex.obtainc C sL | | jd| | s$| t| td|jjtj
|f dS )z-
checker is a ContentChecker
z Validating %%s checksum for %sz7%s validation failed for %s; possible download problem?N) r r r r rU unlinkr r r* r< rQ )r~ checkerrV tfpr! r! r"
check_hash s
zPackageIndex.check_hashc C sN |D ]"}| j du st|r|dstt|r | | q| j | qdS )z;Add `urls` to the list that will be prescanned for searchesNfile:)r r r) r r r append)r~ urlsr8 r! r! r" add_find_links s
zPackageIndex.add_find_linksc C s" | j r tt| j| j d| _ dS )z7Scan urls scheduled for prescanning (e.g. --find-links)N)r r rm r r r! r! r" r s
zPackageIndex.prescanc C s< | |j r | jd}}n| jd}}|||j | d S )Nz#Couldn't retrieve index page for %rz3Couldn't find index page for %r (maybe misspelled?))r r r r r )r~ r methr r! r! r" r s
z PackageIndex.not_found_in_indexc C s~ t |ts5t|}|r)| |d||}t|\}}|dr'| |||}|S tj
|r1|S t |}t |
||ddS )a Locate and/or download `spec` to `tmpdir`, returning a local path
`spec` may be a ``Requirement`` object, or a string containing a URL,
an existing local filename, or a project/version requirement spec
(i.e. the string form of a ``Requirement`` object). If it is the URL
of a .py file with an unambiguous ``#egg=name-version`` tag (i.e., one
that escapes ``-`` as ``_`` throughout), a trivial ``setup.py`` is
automatically created alongside the downloaded file.
If `spec` is a ``Requirement`` object or a string containing a
project/version requirement spec, this method returns the location of
a matching distribution (possibly after downloading it to `tmpdir`).
If `spec` is a locally existing file or directory name, it is simply
returned unchanged. If `spec` is a URL, it is downloaded to a subpath
of `tmpdir`, and the local filename is returned. Various errors may be
raised if a problem occurs during downloading.
r3 r rJ N)r r r
_download_urlrE r@ r( gen_setuprU r< r r# r fetch_distribution)r~ r tmpdirr: foundr+ r? r! r! r" r0 s
zPackageIndex.downloadc s d| i d}d
fdd }|r$ | ||}|s/|dur/|||}|du r@jdur< ||}|du rO|sO| ||}|du r`drZdp[d| dS d| |j|jd S ) a| Obtain a distribution suitable for fulfilling `requirement`
`requirement` must be a ``pkg_resources.Requirement`` instance.
If necessary, or if the `force_scan` flag is set, the requirement is
searched for in the (online) package index as well as the locally
installed packages. If a distribution matching `requirement` is found,
the returned distribution's ``location`` is the value you would have
gotten from calling the ``download()`` method with the matching
distribution's URL or filename. If no matching distribution is found,
``None`` is returned.
If the `source` flag is set, only source distributions and source
checkout links will be considered. Unless the `develop_ok` flag is
set, development and system eggs (i.e., those using the ``.egg-info``
format) will be ignored.
zSearching for %sNc s |d u r}|| j D ];}|jtkr# s#|vr"d| d|< q || v o.|jtkp. }|rF|j}||_tj
|jrF| S q d S )Nz&Skipping development or system egg: %sr3 ) r rA r r r r0 rJ download_locationrU r< r )reqenvrG testloc
develop_okr~ skippedsourcer r! r" rs U s( z-PackageIndex.fetch_distribution..findz:No local packages or working download links found for %s%sza source distribution of r zBest match: %s)rJ ra )r r r r r cloner ) r~ r r
force_scanr r Z local_indexrG rs r! r r" r = s2
z PackageIndex.fetch_distributionc C s" | ||||}|dur|jS dS )a3 Obtain a file suitable for fulfilling `requirement`
DEPRECATED; use the ``fetch_distribution()`` method now instead. For
backward compatibility, this routine is identical but returns the
``location`` of the downloaded distribution instead of a distribution
object.
N)r rJ )r~ r r r r rG r! r! r" fetch s zPackageIndex.fetchc
C s t |}|rdd t||dd D pg }t|dkrxtj|}tj||krEtj ||}ddl
m } |||sEt
|| |}ttj |dd } | d|d j|d jtj|d f W d |S 1 sqw Y |S |rtd ||f td
) Nc S s g | ]}|j r|qS r! )rL )rY dr! r! r"
s z*PackageIndex.gen_setup..r3 r )samefilezsetup.pywzIfrom setuptools import setup
setup(name=%r, version=%r, py_modules=[%r])
zCan't unambiguously interpret project/version identifier %r; any dashes in the name or version should be escaped using underscores. %rzpCan't process plain .py files without an '#egg=name-version' suffix to enable automatic setup script generation.)rC rD r rE rP rU r< rQ dirnamer` Z setuptools.command.easy_installr shutilcopy2r writerK rL splitextr )
r~ rV r? r rD r rQ dstr r r! r! r" r sF
zPackageIndex.gen_setupi c C s2 | d| d }zt|}| |}t|tjjr%td||j |j
f | }d}| j }d}d|v rI| d} t
tt| }| ||||| t|d1}
||} | ro|| |
| |d 7 }| ||||| nnqP| |||
W d n1 sw Y |W |r| S S |r| w w )
NzDownloading %szCan't download %s: %s %sr r/ zcontent-lengthzContent-LengthwbTr3 )r r r r r r4 r r r r r dl_blocksizeget_allmaxrm int
reporthookr r r r r r ) r~ r8 rV fpr r blocknumbssizesizesr r r! r! r" _download_to sF
zPackageIndex._download_toc C s d S ra r! )r~ r8 rV r( Zblksizer* r! r! r" r& zPackageIndex.reporthookc
C s | dr t|S zt|| jW S ttjjfyD } z$ddd |j D }|r0|
|| n t d||f |W Y d }~d S d }~w t j
jyY } z|W Y d }~S d }~w t j
jy } z|rl|
||j n
t d||jf |W Y d }~d S d }~w tjjy } z|r|
||j n
t d||jf |W Y d }~d S d }~w tjjtj
fy } z|r|
|| n t d||f |W Y d }~d S d }~ww )Nr c S s g | ]}t |qS r! )rn )rY argr! r! r" r s z)PackageIndex.open_url..z%s %szDownload error for %s: %sz;%s returned a bad status line. The server might be down, %s)r)
local_openopen_with_authr r httpclient
InvalidURLr` rb r r r4 r r URLErrorreason
BadStatusLineline
HTTPExceptionsocket)r~ r8 warningvr r! r! r" r s^
zPackageIndex.open_urlc C s t |\}}|rd|v r|dddd}d|v s nd}|dr(|d d }tj||}|dks8|d r>| ||S |d
ksG|d rM| ||S |d rX| ||S |d
krht
j t
j
|d S | |d | ||S )Nz...\_Z__downloaded__rH r& svnzsvn+gitzgit+zhg+r r\ T)r@ replacer( rU r< r` r)
_download_svn
_download_git _download_hgr4 r url2pathnamer r5 r _attempt_download)r~ r: r8 r r* r? rV r! r! r" r s(
zPackageIndex._download_urlc C s | |d d S )NT)r r r! r! r" r ) r zPackageIndex.scan_urlc C s2 | ||}d|dd v r| |||S |S )Nr r r )r, r r' _download_html)r~ r8 rV r r! r! r" rG , s z PackageIndex._attempt_downloadc C sj t |}|D ] }| r%td|r#| t| | || S nq| t| td| )Nz ([^- ]+ - )?Revision \d+:z Unexpected HTML page found at ) r ro rX rt r rU r rC r )r~ r8 r rV r r8 r! r! r" rH 3 s
zPackageIndex._download_htmlc C s t dt |ddd }d}| drqd|v rqtj|\}}}}}} |sq|drqd |d
d v rq|d
d d d\}}t |\}
} |
rqd |
v r]|
d d\} }
d | |
f }nd
|
}| }|||||| f}tj
|}| d|| t
d|||f |S )Nz"SVN download support is deprecatedr2 r3 r r zsvn:@z//r. r\ :z --username=%s --password=%sz --username=z'Doing subversion checkout from %s to %szsvn checkout%s -q %s %s)warningsr UserWarningr7 r' r) r4 r r5
_splituser
urlunparser rU system)r~ r8 rV credsr: netlocr< rZ qr authhostuserpwr9 r! r! r" rC B s& zPackageIndex._download_svnc C sp t j| \}}}}}|ddd }|ddd }d }d|v r)|dd\}}t j||||df} | |fS )N+r3 r/ r2 r rI r )r4 r urlsplitr7 rsplit
urlunsplit)r8
pop_prefixr: rQ r< r> r revr! r! r" _vcs_split_rev_from_urlX s z$PackageIndex._vcs_split_rev_from_urlc C l | ddd }| j|dd\}}| d|| td||f |d ur4| d| td ||f |S )
Nr2 r3 r Tr[ z Doing git clone from %s to %szgit clone --quiet %s %szChecking out %sz git -C %s checkout --quiet %sr7 r] r rU rO r~ r8 rV r\ r! r! r" rD j zPackageIndex._download_gitc C r^ )
Nr2 r3 r Tr_ z Doing hg clone from %s to %szhg clone --quiet %s %szUpdating to %szhg --cwd %s up -C -r %s -qr` ra r! r! r" rE z rb zPackageIndex._download_hgc G t j|g|R d S ra )r r r r! r! r" r zPackageIndex.debugc G rc ra )r r r r! r! r" r rd zPackageIndex.infoc G rc ra )r r r r! r! r" r rd zPackageIndex.warn)r r NT)Fra )FFFN)FF))r r r r r r r r r r r r r r r r r r r r r0 r r r r" r, r& r r r rG rH rC staticmethodr] rD rE r r r
__classcell__r! r! r r" r sT
5
#
L
)$
# r z!&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?c C s | d}t|S )Nr )rE r unescape)rD whatr! r! r"
decode_entity s
ri c C s
t t| S )a
Decode HTML entities in the given text.
>>> htmldecode(
... 'https://../package_name-0.1.2.tar.gz'
... '?tokena=A&tokenb=B">package_name-0.1.2.tar.gz')
'https://../package_name-0.1.2.tar.gz?tokena=A&tokenb=B">package_name-0.1.2.tar.gz'
)
entity_subri )textr! r! r" rr s
rr c s fdd}|S )Nc s fdd}|S )Nc s: t }t z
| i |W t | S t | w ra )r: getdefaulttimeoutsetdefaulttimeout)rb rc Z old_timeout)re timeoutr! r" _socket_timeout s
z@socket_timeout.<locals>._socket_timeout.<locals>._socket_timeoutr! )re ro rn rd r" ro s z'socket_timeout.<locals>._socket_timeoutr! )rn ro r! rp r" socket_timeout s rq c C s2 t j| }| }t|}| }|ddS )a9
Encode auth from a URL suitable for an HTTP header.
>>> str(_encode_auth('username%3Apassword'))
'dXNlcm5hbWU6cGFzc3dvcmQ='
Long auth strings should not cause a newline to be inserted.
>>> long_auth = 'username:' + 'password'*10
>>> chr(10) in str(_encode_auth(long_auth))
False
r )r4 r r6 encodebase64 b64encoder rB )rS Zauth_sZ
auth_bytesZ
encoded_bytesencodedr! r! r" _encode_auth s
rw c @ rz )
Credentialz:
A username/password pair. Use like a namedtuple.
c C s || _ || _d S ra usernamepassword)r~ rz r{ r! r! r" r s
zCredential.__init__c c s | j V | jV d S ra ry r r! r! r" __iter__ s zCredential.__iter__c C s dt | S )Nz%(username)s:%(password)s)varsr r! r! r" __str__ s zCredential.__str__N)r r r r r r| r~ r! r! r! r" rx s
rx c @ s0 e Zd Zdd Zedd Zdd Zdd Zd S )
PyPIConfigc C sR t g dd}tj| | tjtjdd}tj |r'|
| dS dS )z%
Load from ~/.pypirc
)rz r{
repositoryr ~z.pypircN) dictfromkeys configparserRawConfigParserr rU r< r`
expanduserr r )r~ defaultsrcr! r! r" r s zPyPIConfig.__init__c s& fdd D }tt j|S )Nc s g | ] } |d r|qS )r )r ro )rY sectionr r! r" r s z2PyPIConfig.creds_by_repository.<locals>.<listcomp>)sectionsr rm _get_repo_cred)r~ Zsections_with_repositoriesr! r r" creds_by_repository s
z PyPIConfig.creds_by_repositoryc C s6 | |d }|t| |d | |d fS )Nr rz r{ )r ro rx )r~ r repor! r! r" r s
zPyPIConfig._get_repo_credc C s* | j D ]
\}}||r| S qdS )z
If the URL indicated appears to be a repository defined in this
config, return the credential for that repository.
N)r itemsr) )r~ r8 r credr! r! r" find_credential s
zPyPIConfig.find_credentialN)r r r r propertyr r r r! r! r! r" r s
r c C s: t j| }|\}}}}}}|drtjd|dv r$t|\} }
nd} | sBt | } | rBt
| } | j | f} t j
dg| R | redt| } ||
||||f}
t j|
}t j|}|d| nt j| }|dt ||}| rt j|j\}}}}}}||kr||
kr||||||f}
t j|
|_|S ) z4Open a urllib2 request, handling HTTP authenticationrJ znonnumeric port: '')r2 httpsNz*Authenticating as %s for %s (from .pypirc)zBasic
Authorizationz
User-Agent)r4 r r5 r( r2 r3 r4 rM r r rn rz r r rw rN r Request
add_header
user_agentr8 )r8 r parsedr: rQ r< paramsr> r rS addressr r r9 r r r' s2h2Zpath2Zparam2Zquery2Zfrag2r! r! r" r1 s8
r1 c C s$ | d\}}} |r|| fS d| fS )zNsplituser('user[:passwd]@host[:port]')
--> 'user[:passwd]', 'host[:port]'.rI N)
rpartition)rT rU delimr! r! r" rM 4 s rM c C s | S ra r! )r8 r! r! r"
fix_sf_url? r- r c C s* t j| \}}}}}}t j|}tj|r t j| S | dr}tj
|r}g }t |D ];} tj || }
| dkrXt
|
d } | } W d n1 sQw Y n tj
|
rb| d7 } |dj| d q0d}
|
j| d |d } d
\}}nd \}}} d d
i}t| }t j| ||||S )z7Read a local path, with special support for directoriesr. z
index.htmlrNz<a href="{name}">{name}</a>)r* zB<html><head><title>{url}{files} |