U
.e( ! @ s d dl mZmZmZ d dlmZmZ d dlmZm Z d dl
Z
d dl Z d dl m
Z
ddlmZmZmZmZ ddlmZ dd lmZ d d
lmZ zd d lmZ W n ek
r eZY nX ed d
eD Zedd
eD Z edd
eD Z eeddgB Z dZ ej r(e !e dd e"d d Z#n
e !e Z#e$ddddddd d d d d d!d"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1d2d3d4d5g Z%e !d6Z&i Z'G d7d8 d8e(Z)d9d: Z*G d;d< d d>e+Z,G d?d@ d@e-Z.G dAdB dBe(Z/G dCdD dDe(Z0dEdF Z1dS )G )absolute_importdivisionunicode_literals) text_type binary_type) http_clienturllibN) webencodings )EOFspaceCharacters asciiLettersasciiUppercase)_ReparseException)_utils)StringIO)BytesIOc C s g | ]}| d qS asciiencode.0item r E/usr/lib/python3.8/site-packages/pip/_vendor/html5lib/_inputstream.py
s r c C s g | ]}| d qS r r r r r r r s c C s g | ]}| d qS r r r r r r r s > .)sumr# r% r r r _bufferedBytes^ s z BufferedStream._bufferedBytesc C s< | j |}| j| | jd d7 < t|| jd< |S r' )r" r2 r# appendr$ r( )r% r1 datar r r r/ a s
zBufferedStream._readStreamc C s |}g }| j d }| j d }|t| jk r|dkr| j| }|t|| kr`|}||| g| _ n"t|| }|t|g| _ |d7 }||||| ||8 }d}q |r|| | d|S )Nr r
)r$ r( r# r6 r/ join)r% r1 ZremainingBytesrvZ bufferIndexZ bufferOffsetZ bufferedDataZ bytesToReadr r r r0 h s$
z BufferedStream._readFromBufferN) __name__
__module__ __qualname____doc__r& r+ r. r2 r5 r/ r0 r r r r r! 9 s r! c K s t | tjs(t | tjjr.t | jtjr.d}n&t| drJt | dt }n
t | t }|rdd |D }|rvt
d| t | f|S t | f|S d S )NFr2 r c S s g | ]}| d r|qS )Z _encoding)endswith)r xr r r r s
z#HTMLInputStream..z3Cannot set an encoding with a unicode input, set %r)
isinstancer Z HTTPResponser ZresponseZaddbasefphasattrr2 r TypeErrorHTMLUnicodeInputStreamHTMLBinaryInputStream)sourcekwargsZ isUnicodeZ encodingsr r r HTMLInputStream s
rI c @ sp e Zd ZdZdZdd Zdd Zdd Zd d
Zd d Z d
d Z
dddZ dd Z dd Z
d ddZdd ZdS ) rE Provides a unicode stream of characters to the HTMLTokenizer.
This class takes care of character encoding and removing or replacing
incorrect byte-sequences and also provides column and line tracking.
i ( c C sZ t jsd| _n tddkr$| j| _n| j| _dg| _tddf| _| || _
| dS ) Initialises the HTMLInputStream.
HTMLInputStream(source, [encoding]) -> Normalized stream from source
for use by html5lib.
source can be either a file-object, local filename or a string.
The optional encoding parameter must be a string that indicates
the encoding. If specified, that encoding will be used,
regardless of any BOM or later declaration (such as in a meta
element)
Nu r
r utf-8certain) r supports_lone_surrogatesreportCharacterErrorsr( characterErrorsUCS4characterErrorsUCS2ZnewLineslookupEncoding charEncoding
openStream
dataStreamreset)r% rG r r r r& s
z HTMLUnicodeInputStream.__init__c C s. d| _ d| _d| _g | _d| _d| _d | _d S )N r )r* chunkSize chunkOffseterrors prevNumLines prevNumCols_bufferedCharacterr4 r r r rV s z HTMLUnicodeInputStream.resetc C s t |dr|}nt|}|S zvProduces a file object from source.
source can be either a file object, local filename or a string.
r2 )rC r r% rG r" r r r rT s
z!HTMLUnicodeInputStream.openStreamc C sT | j }|dd|}| j| }|dd|}|dkr@| j| }n ||d }||fS )N
r r r
)r* countr[ rfindr\ )r% r, r* ZnLinesZ positionLineZ lastLinePosZpositionColumnr r r _position s
z HTMLUnicodeInputStream._positionc C s | | j\}}|d |fS )z:Returns (line, col) of the current position in the stream.r
)rc rY )r% linecolr r r r$ s z HTMLUnicodeInputStream.positionc C s6 | j | jkr| stS | j }| j| }|d | _ |S )zo Read one character from the stream or queue if available. Return
EOF when EOF is reached.
r
)rY rX readChunkr r* )r% rY charr r r rg s
zHTMLUnicodeInputStream.charNc C s |d kr| j }| | j\| _| _d| _d| _d| _| j|}| j rX| j | }d | _ n|s`dS t
|dkrt |d }|dksd| krdkrn n|d | _ |d d }| j r| | |
d d
}|
d d
}|| _t
|| _d S )
NrW r Fr
r
i z
r`
T)_defaultChunkSizerc rX r[ r\ r* rY rU r2 r] r( ordrO replace)r% rX r7 Zlastvr r r rf s0
z HTMLUnicodeInputStream.readChunkc C s( t tt|D ]}| jd qd S )Ninvalid-codepoint)ranger( invalid_unicode_refindallrZ r6 )r% r7 _r r r rP % s z*HTMLUnicodeInputStream.characterErrorsUCS4c C s d}t |D ]}|rqt| }| }t|||d rrt|||d }|tkrl| j
d d}q|dkr|dkr|t |d kr| j
d qd}| j
d qd S )NF rn Tri i r
) rp finditerrl groupstartr ZisSurrogatePairZsurrogatePairToCodepointnon_bmp_invalid_codepointsrZ r6 r( )r% r7 skipmatchZ codepointr) Zchar_valr r r rQ ) s" z*HTMLUnicodeInputStream.characterErrorsUCS2Fc C s zt ||f }W nN tk
r^ ddd |D }|s@d| }td| }t ||f< Y nX g }|| j| j}|dkr| j| jkrqn0| }|| jkr|
| j| j| || _q|
| j| jd | sdqqdd|}|S )z Returns a string of characters from the stream up to but not
including any character in 'characters' or EOF. 'characters' must be
a container that supports the 'in' method and iteration over its
characters.
rW c S s g | ]}d t | qS )z\x%02x)rl )r cr r r r N s z5HTMLUnicodeInputStream.charsUntil..z^%sz[%s]+N) charsUntilRegExKeyErrorr9 recompilery r* rY rX endr6 rf ) r% Z
charactersZoppositecharsZregexr: mr rr r r
charsUntil@ s,
z!HTMLUnicodeInputStream.charsUntilc C s@ |d k r<| j dkr.|| j | _| jd7 _n| j d8 _ d S r' )rY r* rX )r% rg r r r ungeto s
z HTMLUnicodeInputStream.unget)N)F)r; r< r= r> rk r& rV rT rc r$ rg rf rP rQ r r r r r r rE s
&
/rE c @ sL e Zd ZdZdddZdd Zd d
Zdd d Zd
d Zdd Z dd Z
dS )rF rJ N windows-1252Tc C s\ | || _t| | j d| _d| _|| _|| _|| _|| _ || _
| || _ |
dS )rK i d N)rT rawStreamrE r& numBytesMetanumBytesChardetoverride_encodingtransport_encodingsame_origin_parent_encodinglikely_encodingdefault_encodingdetermineEncodingrS rV )r% rG r r r r r Z
useChardetr r r r& s z HTMLBinaryInputStream.__init__c C s& | j d j| jd| _t| d S )Nr rm )rS Z
codec_info streamreaderr rU rE rV r4 r r r rV s zHTMLBinaryInputStream.resetc C sD t |dr|}nt|}z|| W n t|}Y nX |S r^ )rC r r. r+ r! r_ r r r rT s
z HTMLBinaryInputStream.openStreamc C s | df}|d d k r |S t| jdf}|d d k r:|S t| jdf}|d d k rX|S | df}|d d k rt|S t| jdf}|d d k r|d jds|S t| jdf}|d d k r|S |r`zddl m
} W n t k
r Y nvX g }| }|j s,| j
| j}|sq,|| || q| t|jd }| j
d |d k r`|dfS t| jdf}|d d k r|S tddfS )NrM r Z tentativezutf-16)UniversalDetectorencodingr ) detectBOMrR r r detectEncodingMetar name
startswithr Z%pip._vendor.chardet.universaldetectorr ImportErrorZdoner r2 r r6 Zfeedcloseresultr. r )r% ZchardetrS r ZbuffersZdetectorr# r r r r r sP
z'HTMLBinaryInputStream.determineEncodingc C s t |}|d krd S |jdkr(t d}nT|| jd krH| jd df| _n4| jd |df| _| td| jd |f d S )Nutf-16beutf-16lerL r rM z Encoding changed from %s to %s)rR r rS r r. rV r )r% Z newEncodingr r r changeEncoding s
z$HTMLBinaryInputStream.changeEncodingc
C s t jdt jdt jdt jdt jdi}| jd}||dd }d}|sp||}d}|sp||dd }d }|r| j | t
|S | j d
dS dS ) zAttempts to detect at BOM at the start of the stream. If
an encoding can be determined from the BOM return the name of the
encoding otherwise return NonerL r r zutf-32lezutf-32be N rs r ) codecsBOM_UTF8 BOM_UTF16_LE BOM_UTF16_BE BOM_UTF32_LE BOM_UTF32_BEr r2 getr. rR )r% ZbomDictstringr r. r r r r s2
z HTMLBinaryInputStream.detectBOMc C sH | j | j}t|}| j d | }|dk rD|jdkrDtd}|S )z9Report the encoding declared by the meta element
r Nr rL )r r2 r EncodingParserr. getEncodingr rR )r% r# parserr r r r r 9 s z(HTMLBinaryInputStream.detectEncodingMeta)NNNNr T)T) r; r< r= r> r& rV rT r r r r r r r r rF s
*
>"rF c @ s e Zd ZdZdd Zdd Zdd Zdd Zd
d Zd d
Z dd Z
dd Z e e e
Z
dd Ze eZefddZdd Zdd Zdd Zd S )
EncodingByteszString-like object with an associated position and various extra methods
If the position is ever greater than the string length then an exception is
raisedc C s t | | S N)r1 __new__lowerr% valuer r r r L s zEncodingBytes.__new__c C s
d| _ d S )Nr )rc r r r r r& P s zEncodingBytes.__init__c C s | S r r r4 r r r __iter__T s zEncodingBytes.__iter__c C s> | j d }| _ |t| kr"tn |dk r.t| ||d S )Nr
r rc r(
StopIterationrD r% pr r r __next__W s zEncodingBytes.__next__c C s | S r )r r4 r r r next_ s zEncodingBytes.nextc C sB | j }|t| krtn |dk r$t|d | _ }| ||d S r' r r r r r previousc s zEncodingBytes.previousc C s | j t| krt|| _ d S r rc r( r )r% r$ r r r setPositionl s zEncodingBytes.setPositionc C s* | j t| krt| j dkr"| j S d S d S )Nr r r4 r r r getPositionq s
zEncodingBytes.getPositionc C s | | j | j d S Nr
)r$ r4 r r r getCurrentByte{ s z EncodingBytes.getCurrentBytec C sH | j }|t| k r>| ||d }||kr4|| _|S |d7 }q|| _dS )z Skip past a list of charactersr
Nr$ r( rc r% r r rz r r r rx s
zEncodingBytes.skipc C sH | j }|t| k r>| ||d }||kr4|| _|S |d7 }q|| _d S r r r r r r skipUntil s
zEncodingBytes.skipUntilc C s> | j }| ||t| }||}|r:| j t|7 _ |S )zLook for a sequence of bytes at the start of a string. If the bytes
are found return True and advance the position to the byte after the
match. Otherwise return False and leave the position alone)r$ r( r )r% r1 r r7 r: r r r
matchBytes s
zEncodingBytes.matchBytesc C sR | | j d |}|dkrJ| jdkr,d| _| j|t| d 7 _dS tdS )zLook for the next sequence of bytes matching a given sequence. If
a match is found advance the position to the last byte of the matchNr r r
T)r$ findrc r( r )r% r1 Z newPositionr r r jumpTo s
zEncodingBytes.jumpToN)r; r< r= r> r r& r r r r r r propertyr$ r currentBytespaceCharactersBytesrx r r r r r r r r H s
r c @ sX e Zd ZdZdd Zdd Zdd Zdd Zd
d Zd d
Z dd Z
dd Z dd Z dS )r z?Mini parser for detecting character encoding from meta elementsc C s t || _d| _dS )z3string - the data to work on for encoding detectionN)r r7 r r% r7 r r r r& s
zEncodingParser.__init__c
C s d| j fd| jfd| jfd| jfd| jfd| jff}| jD ]Z}d}|D ]D\}}| j|rFz| }W qW qF tk
r d}Y qY qFX qF|s: qq:| jS ) Ns r7 r r4 r r r r s z EncodingParser.handleCommentc C s | j jtkrdS d}d }| }|d kr,dS |d dkr\|d dk}|r|d k r|| _dS q|d dkr|d }t|}|d k r|| _dS q|d dkrtt|d }| }|d k rt|}|d k r|r|| _dS |}qd S ) NTFr s
http-equivr
s content-type charsets content) r7 r r getAttributer rR ContentAttrParserr parse)r% Z hasPragmaZpendingEncodingattrZtentativeEncodingcodecZ
contentParserr r r r s8 zEncodingParser.handleMetac C s
| dS )NF)handlePossibleTagr4 r r r r s z%EncodingParser.handlePossibleStartTagc C s t | j | dS )NT)r r7 r r4 r r r r s
z#EncodingParser.handlePossibleEndTagc C sb | j }|jtkr(|r$| | dS |t}|dkrD| n| }|d k r^| }qLdS )NTr )r7 r asciiLettersBytesr r r spacesAngleBracketsr )r% ZendTagr7 rz r r r r r s
z EncodingParser.handlePossibleTagc C s | j dS )Nr r r4 r r r r s zEncodingParser.handleOtherc C s | j }|ttdgB }|dkr&dS g }g }|dkr>|r>qnX|tkrR| }qnD|dkrhd|dfS |tkr|| n|dkrdS || t|}q.|dkr| d|dfS t| | }|dkr2|}t|}||kr
t| d|d|fS |tkr$|| q|| qnJ|dkrJd|dfS |tkrd|| n|dkrrdS || t|}|t
krd|d|fS |tkr|| n|dkrdS || q|dS ) z_Return a name,value pair for the next attribute in the stream,
if one is found, or None /)r NN =)r r r8 ) ' "r ) r7 rx r frozensetr9 asciiUppercaseBytesr6 r r r r )r% r7 rz ZattrNameZ attrValueZ quoteCharr r r r s`
zEncodingParser.getAttributeN)
r; r< r= r> r& r r r r r r r r r r r r r s $r c @ s e Zd Zdd Zdd ZdS )r c C s
|| _ d S r )r7 r r r r r& f s zContentAttrParser.__init__c C s z| j d | j jd7 _| j | j jdks s
J g Ih 6'