֦i d Z ddlZddlmZ ddlZddlZddlZddlm Z ddl
Z
ddl Z ddl m
Z
ddlZddlZddlmZ e j$ Zda G d d Zd
Z e
j. e d Zd Z G d
de Z G d d Z d Z G d de Z G d de Z G d de Z! G d de Z"d Z#d Z$ d&d Z%d'd Z& G d d e
jN Z(da)da*d Z+d! Z, G d" d#ejZ Z. G d$ d%ej^ Z0y)(z"Brian Quinlan (brian@sweetapp.com) N)_base)Queue)partial)format_exceptionFc $ e Zd Zd Zd Zd Zd Zy)
_ThreadWakeupc d| _ t j | _ t j
d \ | _ | _ y )NF)duplex)_closed threadingLock_lockmpPipe_reader_writerselfs 3/usr/lib64/python3.12/concurrent/futures/process.py__init__z_ThreadWakeup.__init__E s. ^^%
%'WWE%:" dl c | j 5 | j s;d| _ | j j | j j d d d y # 1 sw Y y xY wNT)r r r closer r s r r z_ThreadWakeup.closeJ sC ZZ<< # ""$ ""$ ZZs AA A'c | j 5 | j s| j j d d d d y # 1 sw Y y xY w)Nr )r r r
send_bytesr s r wakeupz_ThreadWakeup.wakeupV s,
ZZ<< '', ZZs (>Ac | j r t d | j j r6| j j | j j r5y y )Nz!operation on closed _ThreadWakeup)r RuntimeErrorr poll
recv_bytesr s r clearz_ThreadWakeup.clear[ sG << BC Cll ! LL # # % ll !r N)__name__
__module__ __qualname__r r r r" r r r r D s ;
%-
&r r c da t t j } | D ] \ }}|j | D ] \ }}|j y r )_global_shutdownlist_threads_wakeupsitemsr join)r+ _
thread_wakeupts r _python_exitr0 b sP !'') *E != " 1 r = c e Zd Zd Zd Zy)_RemoteTracebackc || _ y Ntb)r r8 s r r z_RemoteTraceback.__init__ s r c | j S r6 r7 r s r __str__z_RemoteTraceback.__str__ s wwr N)r# r$ r% r r: r& r r r4 r4 s r r4 c e Zd Zd Zd Zy)_ExceptionWithTracebackc dj t t | || }|| _ d | j _ d|z | _ y )N z
"""
%s""")r, r typeexc
__traceback__r8 )r r@ r8 s r r z _ExceptionWithTraceback.__init__ s?
WW%d3ib9
: "& 2%r c > t | j | j ffS r6 ) _rebuild_excr@ r8 r s r
__reduce__z"_ExceptionWithTraceback.__reduce__ s dhh 000r N)r# r$ r% r rD r& r r r< r< s &1r r< c & t | | _ | S r6 )r4 __cause__)r@ r8 s r rC rC s $R(CM Jr c e Zd Zd Zy) _WorkItemc < || _ || _ || _ || _ y r6 )futurefnargskwargs)r rJ rK rL rM s r r z_WorkItem.__init__ s r Nr# r$ r% r r& r r rH rH r rH c e Zd ZddZy) _ResultItemNc < || _ || _ || _ || _ y r6 )work_id exceptionresultexit_pid)r rS rT rU rV s r r z_ResultItem.__init__ s "
r NNNrN r& r r rQ rQ s !r rQ c e Zd Zd Zy) _CallItemc < || _ || _ || _ || _ y r6 )rS rK rL rM )r rS rK rL rM s r r z_CallItem.__init__ s r NrN r& r r rY rY rO r rY c , e Zd Z d fd Z fdZ xZS )
_SafeQueuec D || _ || _ t |
|| y )N)ctx)pending_work_itemsr. superr )r max_sizer^ r_ r. __class__s r r z_SafeQueue.__init__ s% "4 *
s+r c t |t rt t | ||j }t dj
dj | |_ | j j |j d }| j j | |j j | y y t | E || y )Nz
"""
{}"""r> )
isinstancerY r r? rA r4 formatr, rF r_ poprS r. r rJ
set_exceptionr` _on_queue_feeder_error)r eobjr8 work_itemrb s r rh z!_SafeQueue._on_queue_feeder_error s c9 %!$q'1aoo>B*>+@+@+MNAK //33CKKFI % % ' $ ..q1 %
G *1c 2r )r )r# r$ r% r rh
__classcell__rb s @r r\ r\ s G,
3 3r r\ c ' l K t | } t t j || }|sy | 'wr6 )ziptuple itertoolsislice) chunksize iterablesitchunks r _get_chunksrw s< 6 i B
i&&r956 s 24c 4 |D cg c] } | | c}S c c}w r6 r& )rK rv rL s r _process_chunkry s% #( (%$BI% (( (s c | j t |||| y # t $ r=}t ||j }| j t ||| Y d }~y d }~ww xY w)N)rU rT rV rT rV )putrQ
BaseExceptionr< rA ) result_queuerS rU rT rV ri r@ s r _sendback_resultr sk 89WV/88M N 9%a 9W.68 9 99s " A(3A##A(c D | || d}d } | j d }|$|j t
j y | |dz
}||k\ rt
j } |j |j i |j }t ||j || ~~|y # t $ r$ t j j dd Y y w xY w# t $ r9} t | | j }
t ||j |
| Y d } ~ rd } ~ ww xY w) NzException in initializer:T)exc_infor blockr1 )rU rV r{ )r} r LOGGERcriticalgetr| osgetpidrK rL rM r rS r< rA )
call_queuer~ initializerinitargs max_tasks num_tasksrV call_itemrri r@ s r _process_workerr s8 " IH
NNN. RYY[ ) NII% 99; innA 0@0@AA
\9+<+.weakref_cb4 s# GGMM 1
2 "r ) _executor_manager_thread_wakeupr. _shutdown_lock
shutdown_lockweakrefrefexecutor_reference
_processes processes _call_queuer
_result_queuer~ _work_idswork_ids_queue_max_tasks_per_childmax_tasks_per_child_pending_work_itemsr_ r` r )r executorr rb s r r z _ExecutorManagerThread.__init__& s
& E E % 4 4 &*%7%7 # #*++h
"C ",, #.. % 2 2 ' 0 0 $,#@#@ #+">">
r c | j | j \ }}}|r| j | y || j | |j d u}|r5| j j |j }|j ~| j x}rC|r&| j 5 |j d d d n|j j ~| j r=| j! | j | j" s| j% y 6# t $ r&}t | }| j | Y d }~y d }~ww xY w# 1 sw Y xY wr6 )add_call_item_to_queuer} r terminate_broken wait_result_broken_or_wakeupprocess_result_itemrV r rf r, r r _adjust_process_count_idle_worker_semaphorereleaseis_shutting_downflag_executor_shutting_downr_ join_executor_internals)r r@ cause result_item is_brokenprocess_exitedpr s r runz_ExecutorManagerThread.runS sY
++- -1,M,M,O )KE%%e,&((5!,!5!5T!A! **;+?+?@AFFH # 6 6 8888% ! / / $ : : < 0 / !77??A $$&002
++- ..002_
!
(-%%e,
0 0 /s# D8 4E*8 E' E""E'*E3c | j j ry | j j d }| j | }|j
j
rH| j j t ||j |j |j d n| j |= # t j $ r Y y w xY w)NTFr )r fullr r r_ rJ set_running_or_notify_cancelr| rY rK rL rM queueEmpty)r rS rk s r r z-_ExecutorManagerThread.add_call_item_to_queue s ##%
--111> ! 3 3G < ##@@BOO'' '2;,,2;..2;2B2B)D /3 ( 4 //8 %
;;
s B: :CCc | j j }| j j }||g}t | j j D cg c] }|j }}t j j ||z }d }d}d } ||v r |j } d}n||v rd}| j j | ||fS c c}w # t $ r}
t |
}Y d }
~
=d }
~
ww xY wNTF)r~ r r. r) r valuessentinelr
connectionwaitrecvr} r r" ) r
result_reader
wakeup_readerreadersr worker_sentinelsreadyr r r r@ s r r z3_ExecutorManagerThread.wait_result_broken_or_wakeup s ))11
**22
-004T^^5J5J5L0MN0M1AJJ0MN
""7-=#=> E !
. + 0 0 2 ! e
# I "Iu,,% O !
.(-
.s CC C3 C..C3c t |t rI| j j | }|j | j s| j y y | j j |j d }|X|j r&|j j |j y |j j |j y y r6 )
rd intr rf r, r r_ rS rT rJ rg
set_resultrU )r r r rk s r r z*_ExecutorManagerThread.process_result_item s k3 ' "";/A
FFH>>,,. "
//33K4G4GNI$ (( $$22;3H3HI $$// 0B0BC %r c V | j }t xs |d u xs |j S r6 )r r( _shutdown_thread)r r s r r z'_ExecutorManagerThread.is_shutting_down s4 **,
! -H$4 -,, .r c B | j }|d|_ d|_ d }t d }|#t ddj | d |_ | j j D ]" \ }} |j j | ~$ | j j | j j D ] }|j! | j" j% | j' d y # t j $ r Y w xY w)NzKA child process terminated abruptly, the process pool is not usable anymoreTz^A process in the process pool was terminated abruptly while the future was running or pending.z
'''
r> z''')broken)r _brokenr BrokenProcessPoolr4 r, rF r_ r+ rJ rg r InvalidStateErrorr" r r terminater _terminate_broken_join_executor_internals)r r r bperS rk r s r r z(_ExecutorManagerThread._terminate_broken s! **, !1H )-H %H !6 7 ,"''%.)- /CM #'"9"9"?"?"A GY
..s3 #B
%%' &&(A
KKM )
))+
%%T%2% **
s 4DD D c h | j 5 | j | d d d y # 1 sw Y y xY wr6 )r r )r r s r r z'_ExecutorManagerThread.terminate_broken s%
" "5 )
s (1c d | j }|}d|_ |j rii }| j j D ]% \ }}|j
j
r!|||< ' || _ | j j y y y # t j $ r Y nw xY wd|_ $r ) r r _cancel_pending_futuresr_ r+ rJ cancelr
get_nowaitr r )r r new_pending_work_itemsrS rk s r r z2_ExecutorManagerThread.flag_executor_shutting_down s **, (,H %// *,&*.*A*A*G*G*I&GY$++224:C.w7 +J +A' ++668 0 !;; 490s 3B B'&B'c , | j }d}||k rc| j dkD rOt ||z
D ]# } | j j d |dz
}% ||k r| j dkD rMy y y y # t j
$ r Y 3w xY w)Nr r1 )get_n_children_aliveranger
put_nowaitr Full)r n_children_to_stopn_sentinels_sentis r shutdown_workersz'_ExecutorManagerThread.shutdown_workers* s ! 6 6 8 "44--/!3-0@@AOO..t4$)$ B "44--/!3 53 5 zz s A<<BBc f | j 5 | j d d d y # 1 sw Y y xY wr6 )r r r s r r z._ExecutorManagerThread.join_executor_internals8 s#
) ) +
s '0c F |s| j | j j | j j | j j | j
j
D ]$ }|r|j |j & y r6 ) r r r join_threadr. r r r r, )r r r s r r z/_ExecutorManagerThread._join_executor_internals< ss ! ! #
##% " &&(A
FFH )r c V t d | j j D S )Nc 3 < K | ] }|j y wr6 )is_alive).0r s r z>_ExecutorManagerThread.get_n_children_alive..O s A)@A1::<)@s )sumr r r s r r z+_ExecutorManagerThread.get_n_children_aliveM s A )>)>)@AAAr )F)r# r$ r% r r r r r r r r r r r r r rl rm s @r r r sN +Z2h . -<D..-3^*94 ,"Br r c t rt rt t da dd l} t j d }|dk( ry |dk\ ry d|z at t # t $ r dat t w xY w# t t f$ r Y y w xY w)NTr zxThis Python build lacks multiprocessing.synchronize, usually due to named semaphores being unavailable on this platform.SC_SEM_NSEMS_MAX z@system provides too few semaphores (%d available, 256 necessary)) _system_limits_checked_system_limitedNotImplementedErrormultiprocessing.synchronize ImportErrorr sysconfAttributeError
ValueError)multiprocessing nsems_maxs r _check_system_limitsr V s %o6 6 !3*JJ 12 B C 46?@O
o
..+ 3
F "/22 3
J ' s A A7 A47B B c # l K | D ]* }|j |s|j |r, y wr6 )reverserf )iterableelements r _chain_from_iterable_of_listsr v s5
++- s 444c e Zd Zy)r N)r# r$ r% r& r r r r s r r c e Zd Z ddddZd Zd Zd Zd Zd Ze j j j e_ dd d
fd
Z dd d
dZ
e j j j e
_ xZS )ProcessPoolExecutorN)r c t |Pt j xs d| _ t j
dk( ret
t | j | _ nE|dk r t d t j
dk( r|t kD rt dt || _ |,|t j d }nt j }|| _ | j j d d k7 | _
|t | s t d
|| _ || _ |Ut% |t& s t d |dk r t d | j j d d k( r t d
|| _ d | _ i | _ d| _ t1 j2 | _ t1 j6 d | _ d| _ d| _ i | _ d| _ tC | _" | j tF z }tI || j | j> | jD | _% d| jJ _& |jO | _( tS jT | _+ y )Nr1 win32r z"max_workers must be greater than 0zmax_workers must be <= spawnF)
allow_noneforkz initializer must be a callablez&max_tasks_per_child must be an integerz max_tasks_per_child must be >= 1zpmax_tasks_per_child is incompatible with the 'fork' multiprocessing start method; supply a different mp_context.)ra r^ r_ r. T),r r cpu_count _max_workerssysplatformmin_MAX_WINDOWS_WORKERSr r get_context _mp_contextget_start_method#_safe_to_dynamically_spawn_childrencallable TypeError _initializer _initargsrd r r _executor_manager_threadr r r r
r Semaphorer r _queue_countr r r r EXTRA_QUEUED_CALLSr\ r
_ignore_epipe SimpleQueuer r r r )r max_workers
mp_contextr r r
queue_sizes r r z ProcessPoolExecutor.__init__ s] $ " 3!D ||w&$'(<(,(9(9%;! a !EFF,,') 22 -.B-CDF F !,D ". ^^G 4
^^ -
% 11U1CvM
0 "8K+@ <= = '! * 137 HII$) !CDD 00E0BfL "C D D %8! )-% !& 'nn .&/&9&9!&<# #% ',$ 0=, &&);;
% T%5%5 # 7 7 >> @ *.& ' 3 3 5 r c | j d| j s| j t | | _ | j j | j
t | j < y y r6 )r r _launch_processesr startr r* r s r _start_executor_manager_threadz2ProcessPoolExecutor._start_executor_manager_thread s` ( ( 0;;&&(,B4,HD ) ) ) / / 144
T : : ;
1r c | j j d ry t | j }|| j k r| j y y )NF)blocking)r acquirelenr r
_spawn_process)r
process_counts r r z)ProcessPoolExecutor._adjust_process_count sK & & . . . > DOO,
4,, ,
! -r c t t | j | j D ] }| j y r6 )r r* r r
r+ )r r- s r r$ z%ProcessPoolExecutor._launch_processes s1
s4??+T->->?A ! @r c | j j t | j | j | j
| j | j f }|j || j |j < y )N)targetrL ) r Processr r r r r r r% r pid)r r s r r+ z"ProcessPoolExecutor._spawn_process sm $ $"""$$##..++ -
%
.
!" r c R | j 5 | j rt | j | j r t d t
r t d t
j }t |||| }|| j | j < | j j | j | xj dz
c_
| j j | j r| j! | j# |cd d d S # 1 sw Y y xY w)Nz*cannot schedule new futures after shutdownz6cannot schedule new futures after interpreter shutdownr1 )r r r r r r( r FuturerH r r r r| r r r r r& )r rK rL rM fws r submitzProcessPoolExecutor.submit" s
||' 55$$"#OPP " $: ; ; A!Rv.A:;D $ $T%6%6 7 NN t 0 0 1 " 0 0 7 7 977**, / / 1+ !
s DD D&r1 )timeoutrs c |dk r t d t | t t | t |d|i| }t
| S )Nr1 zchunksize must be >= 1.rs )r7 )r r` mapr ry rw r )r rK r7 rs rt resultsrb s r r9 zProcessPoolExecutor.map; sV ( q= 67 7'+gnb 9 )9 J J&- / -W55r F)cancel_futuresc | j 5 || _ d| _ | j | j j d d d | j
|r| j
j
d | _ d | _ | j |r| j j d | _ d | _
d | _ y # 1 sw Y }xY wr ) r r r r r r r, r r r r )r r r; s r shutdownz ProcessPoolExecutor.shutdownX s
+9D ($(D !33?44;;= ! ( ( 4 ) ) . . 0 )-% )d $ $ & ! /3,# !
s 5B>>C)NNNr& )T)r# r$ r% r r& r r$ r+ r6 r Executor__doc__r9 r= rl rm s @r r r s~ 48,.j'GKj'X5
" " #. ^^**22FN*.! 6:4E 4( ~~..66Hr r rW r6 )1
__author__r concurrent.futuresr r r r multiprocessing.connectionmultiprocessing.queuesr r r functoolsr rq r tracebackr WeakKeyDictionaryr* r( r r0 _register_atexitr r Exceptionr4 r< rC objectrH rQ rY r\ rw ry r r Threadr r r r r BrokenExecutorr r> r r& r r rL s? (T 2
$ " (
& -7,,. & &< < ( y 1 1 !& ! 3 3, ) DH " 93luBY - - uBp /@ ,, c7%.. c7r