U
f37 @ s U d Z ddlmZ ddlZddlZddlmZ ddlmZ ddl m
Z
m Z ddl Z ddl
mZmZmZmZ e
rzddlmZ d d
lmZ d d lmZ d Zd
ZdZdZdZ ee dj ddZ G dd de!Z"G dd de"Z#G dd de"Z$ddd d d Z%G d d d ej&Z'G d!d" d"Z(d#d$d%d&Z)d#d$d'd(Z*d#d$d)d*Z+d#d$d+d,Z,dZ-d-e.d.< e, re* re) se+ se( Z-ndZ-d/d0d1d2d3Z/d4a0d5d$d6d7Z1d4a2d5d$d8d9Z3d/d:d1d;d<Z4d=d$d>d?Z5d#d$d@dAZ6dS )Ba
Support for streaming http requests in emscripten.
A few caveats -
Firstly, you can't do streaming http in the main UI thread, because atomics.wait isn't allowed.
Streaming only works if you're running pyodide in a web worker.
Secondly, this uses an extra web worker and SharedArrayBuffer to do the asynchronous fetch
operation, so it requires that you have crossOriginIsolation enabled, by serving over https
(or from localhost) with the two headers below set:
Cross-Origin-Opener-Policy: same-origin
Cross-Origin-Embedder-Policy: require-corp
You can tell if cross origin isolation is successfully enabled by looking at the global crossOriginIsolated variable in
javascript console. If it isn't, streaming requests will fallback to XMLHttpRequest, i.e. getting the whole
request into a buffer and then returning it. it shows a warning in the javascript console in this case.
Finally, the webworker which does the streaming fetch is created on initial import, but will only be started once
control is returned to javascript. Call `await wait_for_streaming_ready()` to wait for streaming fetch.
NB: in this code, there are a lot of javascript objects. They are named js_*
to make it clear what type of object they are.
) annotationsN)Parser)files)
TYPE_CHECKINGAny)JsArray JsExceptionJsProxyto_js)Buffer )EmscriptenRequest)EmscriptenResponse)z
user-agentzemscripten_fetch_worker.jszutf-8)encodingc s0 e Zd Zd ddddddd fddZ ZS )
_RequestErrorNrequestresponsez
str | NonezEmscriptenRequest | NoneEmscriptenResponse | None)messager r c s$ || _ || _|| _t | j d S N)r r r super__init__)selfr r r __class__ Q/opt/alt/python38/lib/python3.8/site-packages/urllib3/contrib/emscripten/fetch.pyr C s z_RequestError.__init__)N)__name__
__module__ __qualname__r
__classcell__r r r r! r B s
r c @ s e Zd ZdS )_StreamingErrorNr" r# r$ r r r r! r& P s r& c @ s e Zd ZdS )
_TimeoutErrorNr' r r r r! r( T s r( zdict[str, Any]r )dict_valreturnc C s t | tjjdS )N)Zdict_converter)r
jsZObjectZ fromEntries)r) r r r! _obj_from_dictX s r, c s e Zd ZdddddddddZd d
d d Zd
d
ddZed
d
ddZd d
fdd Zd
d
ddZ d
d
ddZ
d
d
ddZ dddd d Z Z
S ) _ReadStreamr floatr intr
)
int_buffer byte_buffertimeoutworker
connection_idr c C sT || _ || _d| _d| _|| _|| _|dkr8td| nd | _d| _d| _ || _
d S )Nr TF) r0 r1 read_posread_lenr4 r3 r/ r2 is_live
_is_closedr )r r0 r1 r2 r3 r4 r r r r! r ] s z_ReadStream.__init__Noner* c C s | d S r )closer r r r! __del__q s z_ReadStream.__del__boolc C s | j S r )r9 r= r r r! is_closedu s z_ReadStream.is_closedc C s | S r )r@ r= r r r! closedy s z_ReadStream.closedc s\ | sXd| _d| _d | _d | _d| _d | _| jrN| j t
d| j i d| _t
d S )Nr Tr< F)r@ r7 r6 r0 r1 r9 r r8 r3 postMessager, r4 r r< r= r r r! r< } s z_ReadStream.closec C s dS NTr r= r r r! readable s z_ReadStream.readablec C s dS NFr r= r r r! writable s z_ReadStream.writablec C s dS rE r r= r r r! seekable s z_ReadStream.seekabler )byte_objr* c C s> | j std| jd d| jdkrtj| j dt | j t
d| j i tj | j dt| j
dkrft| j d }|dkr|| _d| _nZ|tkr| j d }tj }|| jd|}td| | jd dnd| _| dS t| jtt|}| j| j| j| }|t|d|< | j|8 _| j|7 _|S ) Nz,No buffer for stream in _ReadStream.readintor r ZgetMorez timed-outr Exception thrown in fetch: F) r0 r& r r7 r+ Atomicsstore
ERROR_TIMEOUTr3 rB r, r4 waitr2 r( r6 ERROR_EXCEPTION TextDecodernewdecoder1 slicer8 r< minlen
memoryviewsubarrayto_py)r rH data_len
string_len
js_decoderjson_strZ
ret_lengthrV r r r! readinto sN
z_ReadStream.readinto)r" r# r$ r r> r@ propertyrA r< rD rF rG r\ r% r r r r! r- \ s
r- c @ s* e Zd ZddddZddddd Zd
S ) _StreamingFetcherr: r; c sd d _ tjtgtddi}dddd fdd }tj|}tjj | _
tjj | _ d S ) NFtypezapplication/javascriptr r: )
js_resolve_fn js_reject_fnr* c s> dddfdd }ddd fdd }|j _|j _d S )Nr r: )er* c s d_ | d S rC )streaming_readyrb )r` r r r! onMsg s zC_StreamingFetcher.__init__..promise_resolver..onMsgc s | d S r r rd )ra r r! onErr s zC_StreamingFetcher.__init__..promise_resolver..onErr) js_workerZ onmessageonerror)r` ra re rf r= )ra r` r! promise_resolver s z4_StreamingFetcher.__init__..promise_resolver)
rc r+ BlobrP _STREAMING_WORKER_CODEr, URLZcreateObjectURLZ
globalThisWorkerrg ZPromisejs_worker_ready_promise)r Z js_data_blobri Z js_data_urlr r= r! r s
z_StreamingFetcher.__init__r
r r r* c C s dd |j D }|j}|t||jd}|jdkrDtd|j nd }tj d}tj
|}tj |d}tj
|dt tj |d tj |jtjj} | jt|| |d tj |dt| |d tkrtd |d d
n|d tkrP|d }
tj } | | d|
} t | }
t ||
d |
d
t |||j| j|
d |dS |d t!kr|d }
tj } | | d|
} t"d| |d d
nt"d|d |d d
d S )Nc S s i | ]\}}|t kr||qS r )HEADERS_TO_IGNORE).0kvr r r!
s z*_StreamingFetcher.send..)headersbodymethodr r5 i )bufferurlZ fetchParamsz'Timeout connecting to streaming requestr r statusru Z connectionID)r status_coderu rv rI z%Unknown status from worker in fetch: )#ru itemsrv r
rw r2 r/ r+ ZSharedArrayBufferrP Z
Int32ArrayZ
Uint8ArrayrJ rK rL notifyrl rz locationhrefrg rB r, rM r( SUCCESS_HEADERrO rQ rR jsonloadsr r- rN r& )r r ru rv Z
fetch_datar2 Zjs_shared_bufferZ
js_int_bufferZjs_byte_bufferZjs_absolute_urlrY rZ r[ Z response_objr r r! send sv
z_StreamingFetcher.sendN)r" r# r$ r r r r r r! r^ s r^ r? r; c C s t tdo t tdo tjtjkS )Nwindowr )hasattrr+ r r r r r r! is_in_browser_main_thread$ s r c C s t tdotjS )NcrossOriginIsolated)r r+ r r r r r! is_cross_origin_isolated( s r c C s2 t tdo0t tjdo0t tjjdo0tjjjdkS )Nprocessreleasenamenode)r r+ r r r r r r r!
is_in_node, s
r c C s t tdot tdS )Nrm rj )r r+ r r r r! is_worker_available5 s r z_StreamingFetcher | None_fetcherr
r ro c C s" t rt rt | S t d S d S r )r rc r _show_streaming_warningr r r r! send_streaming_requestD s
r Fr: c C s t sda d} tj| d S )NTz8Warning: Timeout is not available on main browser thread)_SHOWN_TIMEOUT_WARNINGr+ consolewarn)r r r r! _show_timeout_warningO s r c C sb t s^da d} t s| d7 } t r(| d7 } t s6| d7 } t dkrH| d7 } dd lm} || d S )
NTz%Can't stream HTTP requests because:
z$ Page is not cross-origin isolated
z+ Python is running in main browser thread
z> Worker or Blob classes are not available in this environment.Fz Streaming fetch worker isn't ready. If you want to be sure that streaming fetch
is working, you need to call: 'await urllib3.contrib.emscripten.fetch.wait_for_streaming_ready()`r )r )_SHOWN_STREAMING_WARNINGr r r rc r+ r r )r r r r r! r Z s
r r c
C sL zt j }t s0d|_| jrFt| jd |_n|d | jrFt | | j
| j d | j
D ] \}}| tkrb||| qb|t| j tt | }t s|j }n |jd}t |j ||| dW S t k
rF } zJ|j dkr t |j!| dn*|j d kr(t"|j!| dnt"|j!| dW 5 d }~X Y nX d S )
NZ arraybufferr5 z text/plain; charset=ISO-8859-15Fz ISO-8859-15)r| ru rv r TimeoutErrorr Z NetworkError)#r+ ZXMLHttpRequestrP r Z responseTyper2 r/ ZoverrideMimeTyper openrw rz ru r} lowerrp ZsetRequestHeaderr r
rv dictr parsestrZgetAllResponseHeadersr rW tobytesencoder r{ r r r( r r )r Zjs_xhrr valueru rv errr r r! send_requestm s<
r z bool | Nonec C s t r
t jS d S d S r )r rc r r r r! rc s rc c s t rt jI d H dS dS d S )NTF)r rn r r r r! wait_for_streaming_ready s r )7__doc__
__future__r ior email.parserr importlib.resourcesr typingr r r+ Z pyodide.ffir r r r
Ztyping_extensionsr r r
r r rp r Z SUCCESS_EOFrL rN __package__joinpath read_textrk Exceptionr r& r( r, RawIOBaser- r^ r r r r r __annotations__r r r r r r rc r r r r r! sf fb )