U
.e( ! @ s d dl mZmZmZ d dlmZmZ d dlmZm Z d dl
Z
d dl Z d dl m
Z
ddlmZmZmZmZ ddlmZ dd lmZ d d
lmZ zd d lmZ W n ek
r eZY nX ed d
eD Zedd
eD Z edd
eD Z eeddgB Z dZ ej rJe d dkr&e !ddks*t"e #e dd e$d d Z%n
e #e Z%e&ddddddd d d d d d!d"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1d2d3d4d5g Z'e #d6Z(i Z)G d7d8 d8e*Z+d9d: Z,G d;d< d d>e-Z.G d?d@ d@e/Z0G dAdB dBe*Z1G dCdD dDe*Z2dEdF Z3dS )G )absolute_importdivisionunicode_literals) text_type binary_type) http_clienturllibN) webencodings )EOFspaceCharacters asciiLettersasciiUppercase)_ReparseException)_utils)StringIO)BytesIOc C s g | ]}| d qS asciiencode.0item r E/usr/lib/python3.8/site-packages/pip/_vendor/html5lib/_inputstream.py
s r c C s g | ]}| d qS r r r r r r r s c C s g | ]}| d qS r r r r r r r s > .)sumr# r% r r r r, ^ s z BufferedStream._bufferedBytesc C s< | j |}| j| | jd d7 < t|| jd< |S r' )r" r4 r# appendr$ r( )r% r3 datar r r r1 a s
zBufferedStream._readStreamc C s |}g }| j d }| j d }|t| jk r|dkr|dks>t| j| }|t|| krl|}||| g| _ n"t|| }|t|g| _ |d7 }||||| ||8 }d}q |r|| | d|S )Nr r
)r$ r( r# r- r7 r1 join)r% r3 ZremainingBytesrvZ bufferIndexZ bufferOffsetZ bufferedDataZ bytesToReadr r r r2 h s&
z BufferedStream._readFromBufferN) __name__
__module__ __qualname____doc__r& r+ r0 r4 r, r1 r2 r r r r r! 9 s r! c K s t | tjs(t | tjjr.t | jtjr.d}n&t| drJt | dt }n
t | t }|rdd |D }|rvt
d| t | f|S t | f|S d S )NFr4 r c S s g | ]}| d r|qS )Z _encoding)endswith)r xr r r r s
z#HTMLInputStream..z3Cannot set an encoding with a unicode input, set %r)
isinstancer Z HTTPResponser ZresponseZaddbasefphasattrr4 r TypeErrorHTMLUnicodeInputStreamHTMLBinaryInputStream)sourcekwargsZ isUnicodeZ encodingsr r r HTMLInputStream s
rJ c @ sp e Zd ZdZdZdd Zdd Zdd Zd d
Zd d Z d
d Z
dddZ dd Z dd Z
d ddZdd ZdS ) rF Provides a unicode stream of characters to the HTMLTokenizer.
This class takes care of character encoding and removing or replacing
incorrect byte-sequences and also provides column and line tracking.
i ( c C sZ t jsd| _n tddkr$| j| _n| j| _dg| _tddf| _| || _
| dS ) Initialises the HTMLInputStream.
HTMLInputStream(source, [encoding]) -> Normalized stream from source
for use by html5lib.
source can be either a file-object, local filename or a string.
The optional encoding parameter must be a string that indicates
the encoding. If specified, that encoding will be used,
regardless of any BOM or later declaration (such as in a meta
element)
Nu r
r utf-8certain) r supports_lone_surrogatesreportCharacterErrorsr( characterErrorsUCS4characterErrorsUCS2ZnewLineslookupEncoding charEncoding
openStream
dataStreamreset)r% rH r r r r& s
z HTMLUnicodeInputStream.__init__c C s. d| _ d| _d| _g | _d| _d| _d | _d S )N r )r* chunkSize chunkOffseterrors prevNumLines prevNumCols_bufferedCharacterr6 r r r rW s z HTMLUnicodeInputStream.resetc C s t |dr|}nt|}|S zvProduces a file object from source.
source can be either a file object, local filename or a string.
r4 )rD r r% rH r" r r r rU s
z!HTMLUnicodeInputStream.openStreamc C sT | j }|dd|}| j| }|dd|}|dkr@| j| }n ||d }||fS )N
r r r
)r* countr\ rfindr] )r% r. r* ZnLinesZ positionLineZ lastLinePosZpositionColumnr r r _position s
z HTMLUnicodeInputStream._positionc C s | | j\}}|d |fS )z:Returns (line, col) of the current position in the stream.r
)rd rZ )r% linecolr r r r$ s z HTMLUnicodeInputStream.positionc C s6 | j | jkr| stS | j }| j| }|d | _ |S )zo Read one character from the stream or queue if available. Return
EOF when EOF is reached.
r
)rZ rY readChunkr r* )r% rZ charr r r rh s
zHTMLUnicodeInputStream.charNc C s |d kr| j }| | j\| _| _d| _d| _d| _| j|}| j rX| j | }d | _ n|s`dS t
|dkrt |d }|dksd| krdkrn n|d | _ |d d }| j r| | |
d d
}|
d d
}|| _t
|| _d S )
NrX r Fr
r
i z
ra
T)_defaultChunkSizerd rY r\ r] r* rZ rV r4 r^ r( ordrP replace)r% rY r8 Zlastvr r r rg s0
z HTMLUnicodeInputStream.readChunkc C s( t tt|D ]}| jd qd S )Ninvalid-codepoint)ranger( invalid_unicode_refindallr[ r7 )r% r8 _r r r rQ % s z*HTMLUnicodeInputStream.characterErrorsUCS4c C s d}t |D ]}|rqt| }| }t|||d rrt|||d }|tkrl| j
d d}q|dkr|dkr|t |d kr| j
d qd}| j
d qd S )NF ro Trj i r
) rq finditerrm groupstartr ZisSurrogatePairZsurrogatePairToCodepointnon_bmp_invalid_codepointsr[ r7 r( )r% r8 skipmatchZ codepointr) Zchar_valr r r rR ) s" z*HTMLUnicodeInputStream.characterErrorsUCS2Fc
C s zt ||f }W nh tk
rx |D ]}t|dk s$tq$ddd |D }|sZd| }td| }t ||f< Y nX g }|| j| j }|dkr| j | j
krqn0| }|| j
kr| | j| j | || _ q| | j| j d |
s~qq~d|} | S )z Returns a string of characters from the stream up to but not
including any character in 'characters' or EOF. 'characters' must be
a container that supports the 'in' method and iteration over its
characters.
rX c S s g | ]}d t | qS )z\x%02x)rm )r cr r r r N s z5HTMLUnicodeInputStream.charsUntil..z^%sz[%s]+N)charsUntilRegExKeyErrorrm r- r: recompilerz r* rZ rY endr7 rg )
r% Z
charactersZoppositecharsr| Zregexr; mr rr r r
charsUntil@ s0
z!HTMLUnicodeInputStream.charsUntilc C sT |d k rP| j dkr.|| j | _| jd7 _n"| j d8 _ | j| j |ksPtd S r' )rZ r* rY r- )r% rh r r r ungeto s
z HTMLUnicodeInputStream.unget)N)F)r< r= r> r? rl r& rW rU rd r$ rh rg rQ rR r r r r r r rF s
&
/rF c @ sL e Zd ZdZdddZdd Zd d
Zdd d Zd
d Zdd Z dd Z
dS )rG rK N windows-1252Tc C sn | || _t| | j d| _d| _|| _|| _|| _|| _ || _
| || _ | j d dk sbt
| dS )rL i d r N)rU rawStreamrF r& numBytesMetanumBytesChardetoverride_encodingtransport_encodingsame_origin_parent_encodinglikely_encodingdefault_encodingdetermineEncodingrT r- rW )r% rH r r r r r Z
useChardetr r r r& s z HTMLBinaryInputStream.__init__c C s& | j d j| jd| _t| d S )Nr rn )rT Z
codec_info streamreaderr rV rF rW r6 r r r rW s zHTMLBinaryInputStream.resetc C sD t |dr|}nt|}z|| W n t|}Y nX |S r_ )rD r r0 r+ r! r` r r r rU s
z HTMLBinaryInputStream.openStreamc C s | df}|d d k r |S t| jdf}|d d k r:|S t| jdf}|d d k rX|S | df}|d d k rt|S t| jdf}|d d k r|d jds|S t| jdf}|d d k r|S |rpzddl m
} W n t k
r Y nX g }| }|j s<| j
| j}t|ts t|s&q<|| || q| t|jd }| j
d |d k rp|dfS t| jdf}|d d k r|S tddfS )NrN r Z tentativezutf-16)UniversalDetectorencodingr ) detectBOMrS r r detectEncodingMetar name
startswithr Z%pip._vendor.chardet.universaldetectorr ImportErrorZdoner r4 r rB r3 r- r7 Zfeedcloseresultr0 r )r% ZchardetrT r ZbuffersZdetectorr# r r r r r sR
z'HTMLBinaryInputStream.determineEncodingc C s | j d dkstt|}|d kr&d S |jdkrFtd}|d k stnT|| j d krf| j d df| _ n4| jd |df| _ | td| j d |f d S )Nr
rN utf-16beutf-16lerM r z Encoding changed from %s to %s)rT r- rS r r r0 rW r )r% Z newEncodingr r r changeEncoding s
z$HTMLBinaryInputStream.changeEncodingc
C s t jdt jdt jdt jdt jdi}| jd}t|t s r? r& rW rU r r r r r r r r rG s
*
>"rG c @ s e Zd ZdZdd Zdd Zdd Zdd Zd
d Zd d
Z dd Z
dd Z e e e
Z
dd Ze eZefddZdd Zdd Zdd Zd S )
EncodingByteszString-like object with an associated position and various extra methods
If the position is ever greater than the string length then an exception is
raisedc C s t |tstt| | S N)rB r3 r- __new__lowerr% valuer r r r L s zEncodingBytes.__new__c C s
d| _ d S )Nr )rd r r r r r& P s zEncodingBytes.__init__c C s | S r r r6 r r r __iter__T s zEncodingBytes.__iter__c C s> | j d }| _ |t| kr"tn |dk r.t| ||d S )Nr
r rd r(
StopIterationrE r% pr r r __next__W s zEncodingBytes.__next__c C s | S r )r r6 r r r next_ s zEncodingBytes.nextc C sB | j }|t| krtn |dk r$t|d | _ }| ||d S r' r r r r r previousc s zEncodingBytes.previousc C s | j t| krt|| _ d S r rd r( r )r% r$ r r r setPositionl s zEncodingBytes.setPositionc C s* | j t| krt| j dkr"| j S d S d S )Nr r r6 r r r getPositionq s
zEncodingBytes.getPositionc C s | | j | j d S Nr
)r$ r6 r r r getCurrentByte{ s z EncodingBytes.getCurrentBytec C sH | j }|t| k r>| ||d }||kr4|| _|S |d7 }q|| _dS )z Skip past a list of charactersr
Nr$ r( rd r% r r r| r r r ry s
zEncodingBytes.skipc C sH | j }|t| k r>| ||d }||kr4|| _|S |d7 }q|| _d S r r r r r r skipUntil s
zEncodingBytes.skipUntilc C s> | j }| ||t| }||}|r:| j t|7 _ |S )zLook for a sequence of bytes at the start of a string. If the bytes
are found return True and advance the position to the byte after the
match. Otherwise return False and leave the position alone)r$ r( r )r% r3 r r8 r; r r r
matchBytes s
zEncodingBytes.matchBytesc C sR | | j d |}|dkrJ| jdkr,d| _| j|t| d 7 _dS tdS )zLook for the next sequence of bytes matching a given sequence. If
a match is found advance the position to the last byte of the matchNr r r
T)r$ findrd r( r )r% r3 Z newPositionr r r jumpTo s
zEncodingBytes.jumpToN)r< r= r> r? r r& r r r r r r propertyr$ r currentBytespaceCharactersBytesry r r r r r r r r H s
r c @ sX e Zd ZdZdd Zdd Zdd Zdd Zd
d Zd d
Z dd Z
dd Z dd Z dS )r z?Mini parser for detecting character encoding from meta elementsc C s t || _d| _dS )z3string - the data to work on for encoding detectionN)r r8 r r% r8 r r r r& s
zEncodingParser.__init__c
C s d| j fd| jfd| jfd| jfd| jfd| jff}| jD ]Z}d}|D ]D\}}| j|rFz| }W qW qF tk
r d}Y qY qFX qF|s: qq:| jS ) Ns r8 r r6 r r r r s z EncodingParser.handleCommentc C s | j jtkrdS d}d }| }|d kr,dS |d dkr\|d dk}|r|d k r|| _dS q|d dkr|d }t|}|d k r|| _dS q|d dkrtt|d }| }|d k rt|}|d k r|r|| _dS |}qd S ) NTFr s
http-equivr
s content-type charsets content) r8 r r getAttributer rS ContentAttrParserr parse)r% Z hasPragmaZpendingEncodingattrZtentativeEncodingcodecZ
contentParserr r r r s8 zEncodingParser.handleMetac C s
| dS )NF)handlePossibleTagr6 r r r r s z%EncodingParser.handlePossibleStartTagc C s t | j | dS )NT)r r8 r r6 r r r r s
z#EncodingParser.handlePossibleEndTagc C sb | j }|jtkr(|r$| | dS |t}|dkrD| n| }|d k r^| }qLdS )NTr )r8 r asciiLettersBytesr r r spacesAngleBracketsr )r% ZendTagr8 r| r r r r r s
z EncodingParser.handlePossibleTagc C s | j dS )Nr r r6 r r r r s zEncodingParser.handleOtherc C s | j }|ttdgB }|dks2t|dks2t|dkr>dS g }g }|dkrV|rVqnX|tkrj| }qnD|dkrd|dfS |tkr|| n|dkrdS || t
|}qF|dkr| d|dfS t
| | }|dkrJ|}t
|}||kr"t
| d|d|fS |tkr<|| q|| qnJ|d krbd|dfS |tkr||| n|dkrdS || t
|}|t krd|d|fS |tkr|| n|dkrdS || qdS )
z_Return a name,value pair for the next attribute in the stream,
if one is found, or None /Nr
)r N =)r r r9 ) ' "r )
r8 ry r frozensetr( r- r: asciiUppercaseBytesr7 r r r r )r% r8 r| ZattrNameZ attrValueZ quoteCharr r r r sb
zEncodingParser.getAttributeN)
r< r= r> r? r& r r r r r r r r r r r r r s $r c @ s e Zd Zdd Zdd ZdS )r c C s t |tst|| _d S r )rB r3 r- r8 r r r r r& f s zContentAttrParser.__init__c C s z| j d | j jd7 _| j | j jdks r& r r r r r r e s r c C sf t | tr0z| d} W n tk
r. Y dS X | dk r^z t| W S tk
rZ Y dS X ndS dS )z{Return the python codec name corresponding to an encoding or None if the
string doesn't correspond to a valid encoding.r N)rB r decodeUnicodeDecodeErrorr lookupAttributeError)r r r r rS s
rS )4Z
__future__r r r Zpip._vendor.sixr r Zpip._vendor.six.movesr r r r Z pip._vendorr Z constantsr r r
r r rX r ior r r r r r r r Z invalid_unicode_no_surrogaterO rb r- r evalrq setrx Zascii_punctuation_rer} objectr! rJ rF rG r3 r r r rS r r r r s
"
J g Ih 6'