ó VèZc@sNdgZddlZddlZddlZddlZddlZddlmZmZm Z ddl m Z m Z dZ dZdZejƒZd„Zd efd „ƒYZefed „Zdefd „ƒYZd efd„ƒYZdefd„ƒYZdefd„ƒYZdefd„ƒYZdefd„ƒYZdS(tPooliÿÿÿÿN(tProcesst cpu_countt TimeoutError(tFinalizetdebugiiicCs t|ŒS(N(tmap(targs((s*/usr/lib/python2.7/multiprocessing/pool.pytmapstar@stMaybeEncodingErrorcBs)eZdZd„Zd„Zd„ZRS(sVWraps possible unpickleable errors, so they can be safely sent through the socket.cCsAt|ƒ|_t|ƒ|_tt|ƒj|j|jƒdS(N(treprtexctvaluetsuperR t__init__(tselfR R ((s*/usr/lib/python2.7/multiprocessing/pool.pyRKscCsd|j|jfS(Ns(Error sending result: '%s'. Reason: '%s'(R R (R((s*/usr/lib/python2.7/multiprocessing/pool.pyt__str__Ps cCsdt|ƒS(Ns(tstr(R((s*/usr/lib/python2.7/multiprocessing/pool.pyt__repr__Ts(t__name__t __module__t__doc__RRR(((s*/usr/lib/python2.7/multiprocessing/pool.pyR Gs  c CsÑ|dks0t|ƒtkr*|dks0t‚|j}|j}t|dƒrn|jjƒ|j jƒn|dk r‡||Œnd}x/|dks®|r¾||kr¾y |ƒ}Wn"t t fk rßt dƒPnX|dkrút dƒPn|\} } } } } yt | | | Žf}Wntk rI}t|f}nXy|| | |fƒWnMtk r°}t||dƒ}t d|ƒ|| | t|ffƒnX|d7}qWt d|ƒdS(Nit_writers)worker got EOFError or IOError -- exitingsworker got sentinel -- exitingis0Possible encoding error while sending result: %ssworker exiting after %d tasks(tNonettypetinttAssertionErrortputtgetthasattrRtcloset_readertEOFErrortIOErrorRtTruet ExceptiontFalseR (tinqueuetoutqueuet initializertinitargstmaxtasksRRt completedttasktjobtitfuncRtkwdstresulttetwrapped((s*/usr/lib/python2.7/multiprocessing/pool.pytworkerXs@0     !    cBseZdZeZddddd„Zd„Zd„Zd„Zd„Z did„Z dd„Z dd „Z dd „Z didd „Zddd „Zed „ƒZed„ƒZed„ƒZed„ƒZd„Zd„Zd„Zd„Zed„ƒZed„ƒZRS(sH Class which supports an async version of the `apply()` builtin c Cs)|jƒtjƒ|_i|_t|_||_||_||_|dkr|y t ƒ}Wq|t k rxd}q|Xn|dkr—t dƒ‚n|dk rÂt |dƒ rÂtdƒ‚n||_g|_|jƒtjdtjd|fƒ|_t|j_t|j_|jjƒtjdtjd|j|j|j|jfƒ|_t|j_t|j_|jjƒtjdtjd|j|j|jfƒ|_ t|j _t|j _|j jƒt!||j"d|j|j#|j|j|j|j|j |jfddƒ|_$dS( Nis&Number of processes must be at least 1t__call__sinitializer must be a callablettargetRt exitpriorityi(%t _setup_queuestQueuet _taskqueuet_cachetRUNt_statet_maxtasksperchildt _initializert _initargsRRtNotImplementedErrort ValueErrorRt TypeErrort _processest_poolt_repopulate_poolt threadingtThreadRt_handle_workerst_worker_handlerR"tdaemontstartt _handle_taskst _quick_putt _outqueuet _task_handlert_handle_resultst _quick_gett_result_handlerRt_terminate_poolt_inqueuet _terminate(Rt processesR'R(tmaxtasksperchild((s*/usr/lib/python2.7/multiprocessing/pool.pyRˆsV                     $          cCswt}xjttt|jƒƒƒD]M}|j|}|jdk r"td|ƒ|jƒt }|j|=q"q"W|S(sCleanup after any worker processes which have exited due to reaching their specified lifetime. Returns True if any workers were cleaned up. scleaning up worker %dN( R$treversedtrangetlenRDtexitcodeRRtjoinR"(RtcleanedR-R3((s*/usr/lib/python2.7/multiprocessing/pool.pyt_join_exited_workersÂs"  c Cs¦xŸt|jt|jƒƒD]}|jdtd|j|j|j|j |j fƒ}|jj |ƒ|j j ddƒ|_ t|_|jƒtdƒqWdS(s€Bring the number of pool processes up to the specified number, for use after reaping workers which have exited. R5RRt PoolWorkers added workerN(RYRCRZRDRR3RTRNR>R?R=tappendtnametreplaceR"RJRKR(RR-tw((s*/usr/lib/python2.7/multiprocessing/pool.pyREÑs#   cCs|jƒr|jƒndS(sEClean up any exited workers and start replacements for them. N(R^RE(R((s*/usr/lib/python2.7/multiprocessing/pool.pyt_maintain_poolás cCsPddlm}|ƒ|_|ƒ|_|jjj|_|jjj|_ dS(Ni(t SimpleQueue( tqueuesReRTRNRtsendRMRtrecvRQ(RRe((s*/usr/lib/python2.7/multiprocessing/pool.pyR7çs   cCs.|jtkst‚|j|||ƒjƒS(s1 Equivalent of `apply()` builtin (R<R;Rt apply_asyncR(RR.RR/((s*/usr/lib/python2.7/multiprocessing/pool.pytapplyîscCs.|jtkst‚|j|||ƒjƒS(s/ Equivalent of `map()` builtin (R<R;Rt map_asyncR(RR.titerablet chunksize((s*/usr/lib/python2.7/multiprocessing/pool.pyRõsicsÝ|jtkst‚|dkrft|jƒ‰|jj‡‡fd†t|ƒDƒˆjfƒˆS|dksxt‚t j ˆ||ƒ}t|jƒ‰|jj‡fd†t|ƒDƒˆjfƒd„ˆDƒSdS(sZ Equivalent of `itertools.imap()` -- can be MUCH slower than `Pool.map()` ic3s0|]&\}}ˆj|ˆ|fifVqdS(N(t_job(t.0R-tx(R.R0(s*/usr/lib/python2.7/multiprocessing/pool.pys sc3s0|]&\}}ˆj|t|fifVqdS(N(RnR(RoR-Rp(R0(s*/usr/lib/python2.7/multiprocessing/pool.pys scss"|]}|D] }|Vq qdS(N((Rotchunktitem((s*/usr/lib/python2.7/multiprocessing/pool.pys sN( R<R;Rt IMapIteratorR:R9Rt enumeratet _set_lengthRt _get_tasks(RR.RlRmt task_batches((R.R0s*/usr/lib/python2.7/multiprocessing/pool.pytimapüs csÝ|jtkst‚|dkrft|jƒ‰|jj‡‡fd†t|ƒDƒˆjfƒˆS|dksxt‚t j ˆ||ƒ}t|jƒ‰|jj‡fd†t|ƒDƒˆjfƒd„ˆDƒSdS(sK Like `imap()` method but ordering of results is arbitrary ic3s0|]&\}}ˆj|ˆ|fifVqdS(N(Rn(RoR-Rp(R.R0(s*/usr/lib/python2.7/multiprocessing/pool.pys sc3s0|]&\}}ˆj|t|fifVqdS(N(RnR(RoR-Rp(R0(s*/usr/lib/python2.7/multiprocessing/pool.pys scss"|]}|D] }|Vq qdS(N((RoRqRr((s*/usr/lib/python2.7/multiprocessing/pool.pys sN( R<R;RtIMapUnorderedIteratorR:R9RRtRuRRv(RR.RlRmRw((R.R0s*/usr/lib/python2.7/multiprocessing/pool.pytimap_unordereds cCsV|jtkst‚t|j|ƒ}|jj|jd|||fgdfƒ|S(s> Asynchronous equivalent of `apply()` builtin N( R<R;Rt ApplyResultR:R9RRnR(RR.RR/tcallbackR0((s*/usr/lib/python2.7/multiprocessing/pool.pyRi s+csû|jtkst‚t|dƒs3t|ƒ}n|dkr}tt|ƒt|jƒdƒ\}}|r}|d7}q}nt|ƒdkr˜d}nt j |||ƒ}t |j |t|ƒ|ƒ‰|j j‡fd†t|ƒDƒdfƒˆS(s< Asynchronous equivalent of `map()` builtin t__len__iiic3s0|]&\}}ˆj|t|fifVqdS(N(RnR(RoR-Rp(R0(s*/usr/lib/python2.7/multiprocessing/pool.pys :sN(R<R;RRtlistRtdivmodRZRDRRvt MapResultR:R9RRt(RR.RlRmR|textraRw((R0s*/usr/lib/python2.7/multiprocessing/pool.pyRk)s ( cCsotjƒ}xB|jtks6|jrP|jtkrP|jƒtjdƒqW|j j dƒt dƒdS(Ngš™™™™™¹?sworker handler exiting( RFtcurrent_threadR<R;R:t TERMINATERdttimetsleepR9RRR(tpooltthread((s*/usr/lib/python2.7/multiprocessing/pool.pyRH>s  * c Cs1tjƒ}x¶t|jdƒD]˜\}}d}x‚t|ƒD]P\}}|jratdƒPny||ƒWq>tk rtdƒPq>Xq>W|rtdƒ||dƒqqPqWtdƒy@tdƒ|j dƒtdƒx|D]} |dƒqðWWntk r"td ƒnXtd ƒdS( Niÿÿÿÿs'task handler found thread._state != RUNscould not put task on queuesdoing set_length()istask handler got sentinels/task handler sending sentinel to result handlers(task handler sending sentinel to workerss/task handler got IOError when sending sentinelsstask handler exiting( RFR‚titerRRRtR<RR!R( t taskqueueRR&R†R‡ttaskseqt set_lengthR-R+tp((s*/usr/lib/python2.7/multiprocessing/pool.pyRLKs6             cCsòtjƒ}x¶y |ƒ}Wn"ttfk r@tdƒdSX|jrm|jtks_t‚tdƒPn|dkr‡tdƒPn|\}}}y||j ||ƒWqt k rÁqXqx¢|ri|jtkriy |ƒ}Wn"ttfk rtdƒdSX|dkr+tdƒqÈn|\}}}y||j ||ƒWqÈt k reqÈXqÈWt |dƒrÕtdƒy5x.t dƒD] }|j jƒs¬Pn|ƒq“WWqÕttfk rÑqÕXntdt|ƒ|jƒdS( Ns.result handler got EOFError/IOError -- exitings,result handler found thread._state=TERMINATEsresult handler got sentinels&result handler ignoring extra sentinelRs"ensuring that outqueue is not fulli s7result handler exiting: len(cache)=%s, thread._state=%s(RFR‚R!R RR<RƒRRt_settKeyErrorRRYRtpollRZ(R&RtcacheR‡R+R,R-tobj((s*/usr/lib/python2.7/multiprocessing/pool.pyRPrsX              ccsCt|ƒ}x0ttj||ƒƒ}|s1dS||fVqdS(N(Rˆttuplet itertoolstislice(R.tittsizeRp((s*/usr/lib/python2.7/multiprocessing/pool.pyRv¬s  cCstdƒ‚dS(Ns:pool objects cannot be passed between processes or pickled(R@(R((s*/usr/lib/python2.7/multiprocessing/pool.pyt __reduce__µscCs5tdƒ|jtkr1t|_t|j_ndS(Ns closing pool(RR<R;tCLOSERI(R((s*/usr/lib/python2.7/multiprocessing/pool.pyRºs  cCs-tdƒt|_t|j_|jƒdS(Nsterminating pool(RRƒR<RIRU(R((s*/usr/lib/python2.7/multiprocessing/pool.pyt terminateÀs   cCsntdƒ|jttfks%t‚|jjƒ|jjƒ|jjƒx|j D]}|jƒqVWdS(Ns joining pool( RR<R˜RƒRRIR\RORRRD(RRŒ((s*/usr/lib/python2.7/multiprocessing/pool.pyR\Æs    cCsWtdƒ|jjƒx9|jƒrR|jjƒrR|jjƒtjdƒqWdS(Ns7removing tasks from inqueue until task handler finishedi( Rt_rlocktacquiretis_aliveRRRhR„R…(R%t task_handlerR–((s*/usr/lib/python2.7/multiprocessing/pool.pyt_help_stuff_finishÏs    c Csµtdƒt|_t|_tdƒ|j||t|ƒƒ|jƒsct|ƒdksct‚t|_|jdƒtdƒt j ƒ|k r¥|j dƒn|røt |ddƒrøtdƒx-|D]"} | j dkrÏ| jƒqÏqÏWntdƒt j ƒ|k r$|j dƒntd ƒt j ƒ|k rP|j dƒn|r±t |ddƒr±td ƒx;|D]0} | jƒrztd | jƒ| j ƒqzqzWndS( Nsfinalizing pools&helping task handler/workers to finishisjoining worker handlerg}Ô%­I²TR™sterminating workerssjoining task handlersjoining result handlersjoining pool workersscleaning up worker %d(RRƒR<RžRZRœRRRRFR‚R\RR[R™tpid( tclsR‰R%R&R†tworker_handlerRtresult_handlerRRŒ((s*/usr/lib/python2.7/multiprocessing/pool.pyRSØs8    $          N((((RRRRRRR^RERdR7RjRRxRzRiRkt staticmethodRHRLRPRvR—RR™R\Ržt classmethodRS(((s*/usr/lib/python2.7/multiprocessing/pool.pyR‚s0  9         ':     R{cBsDeZd„Zd„Zd„Zdd„Zdd„Zd„ZRS(cCsStjtjƒƒ|_tjƒ|_||_t|_ ||_ |||js      cCs|\}}|r¹||j||j|d|j+|jd8_|jdkr|jrn|j|jƒn|j|j=|jjƒzt|_ |jj ƒWd|jj ƒXqnWt |_ ||_|j|j=|jjƒzt|_ |jj ƒWd|jj ƒXdS(Nii(R²R´RµR«R:RnR§R›R"RªR³R°R$R­(RR-tsuccess_resulttsuccessR0((s*/usr/lib/python2.7/multiprocessing/pool.pyRJs* "         (RRRR(((s*/usr/lib/python2.7/multiprocessing/pool.pyR€<s RscBs>eZd„Zd„Zdd„ZeZd„Zd„ZRS(cCsktjtjƒƒ|_tjƒ|_||_tj ƒ|_ d|_ d|_ i|_|||j#s*        *ÿ‰2-I