
    0FieR              
         U d Z ddlmZ ddlZddlZddlZddlZddlZddlm	Z	m
Z
mZmZ ddlmZ ddlmZ ddlmZmZmZmZmZ ddlZddlZddlmZ dd	lmZ dd
lmZm Z m!Z! ddl"m#Z# ddl$m%Z%m&Z& ddl'm(Z( ddl)m*Z*m+Z+ ddl,m-Z-m.Z. ddl/m0Z0m1Z1m2Z2 ddl3m4Z4 erddl5m6Z6 ddl7m8Z8 ddl9m:Z: eeee;f         e	g eee;f         f         e	e<geee;f         f         e=e	deee;f         f         e>e<ef         f         df         Z?de@d<    ejA        d          ZBeBC                     e0d                      ejA        d          ZD G d d          ZE G d d          ZFefd9d(ZGd:d/ZHd;d4ZI G d5 d6          ZJ G d7 d8          ZKdS )<a  Encapsulated manager for in-memory tasks on a worker.

This module covers:
- spill/unspill data depending on the 'distributed.worker.memory.target' threshold
- spill/unspill data depending on the 'distributed.worker.memory.spill' threshold
- pause/unpause the worker depending on the 'distributed.worker.memory.pause' threshold
- kill the worker depending on the 'distributed.worker.memory.terminate' threshold

This module does *not* cover:
- Changes in behaviour in Worker, Scheduler, task stealing, Active Memory Manager, etc.
  caused by the Worker being in paused status
- Worker restart after it's been killed
- Scheduler-side heuristics regarding memory usage, e.g. the Active Memory Manager

See also:
- :mod:`distributed.spill`, which implements the spill-to-disk mechanism and is wrapped
  by this module. Unlike this module, :mod:`distributed.spill` is agnostic to the
  Worker.
- :mod:`distributed.active_memory_manager`, which runs on the scheduler side
    )annotationsN)Callable	ContainerHashableMutableMapping)suppress)partial)TYPE_CHECKINGAnyLiteralUnioncast)	CPU_COUNT)Key)format_bytesparse_bytesparse_timedelta)system)WINDOWSPeriodicCallback)Status)context_meter	monotonic)ManualEvictProtoSpillBuffer)RateLimiterFilterhas_arg
log_errors)ThrottledGC)	TypeAlias)Nanny)Worker.r    WorkerDataParameterzdistributed.worker.memoryzUnmanaged memory use is highzdistributed.nanny.memoryc                      e Zd ZU dZded<   ded<   ded<   ded<   ded	<   d
ed<   ded<   ded<   ddddddd(dZed)d            Zd*dZd*d Z	d*d!Z
d"d#d+d'ZdS ),WorkerMemoryManagera  Management of worker memory usage

    Parameters
    ----------
    worker
        Worker to manage

    For meaning of the remaining parameters, see the matching
    parameter names in :class:`~.distributed.worker.Worker`.

    Notes
    -----

    If data is a callable and has the argument ``worker_local_directory`` in its
    signature, it will be filled with the worker's attr:``local_directory``.

    zMutableMapping[Key, object]data
int | Nonememory_limitfloat | Literal[False]memory_target_fractionmemory_spill_fractionmemory_pause_fractionzint | Literal[False]	max_spillfloatmemory_monitor_intervalr   _throttled_gcautoN)r(   r&   r*   r+   r,   workerr"   nthreadsintstr | floatr#   float | Literal[False] | Nonec               T   t          ||t                    | _        t          dd|          | _        t          dd|          | _        t          dd|          | _        t          j        	                    d          }|d	u rd	nt          |          | _        t          |t                    r	|| _        ndt          |          rWt!          |d
          r't#          d|          } ||j                  | _        nt#          d|          } |            | _        nt          |t&                    rV|\  }	}
t          |	          st)          d          t!          |	d
          r |	dd
|j        i|
| _        n |	di |
| _        n| j        r| j        s| j        rw| j        r$t+          | j        | j        p| j        z            }nt,          j        }t1          t2          j                            |j        d          || j                  | _        ni | _        t          | j        t                    st9          d| j                   | j        rt)          d          t;          t          j        	                    d          d	          | _        t          | j        t*          t>          f          sJ | j        rP| j        d	us	| j        d	ur>| j        J tA          tC          | j"        |          | j        dz            }||j#        d<   tI          t                    | _%        d S )Nloggerz distributed.worker.memory.targetr*   zdistributed.worker.memory.spillr+   zdistributed.worker.memory.pauser,   z#distributed.worker.memory.max-spillFworker_local_directoryz,Callable[[str], MutableMapping[Key, object]]z)Callable[[], MutableMapping[Key, object]]zExpecting a callablestorage)targetr-   z*Worker.data must be a MutableMapping; got z0Worker.data must be empty at initialization time*distributed.worker.memory.monitor-intervaldefault  memory_monitor )&parse_memory_limitworker_loggerr(   _parse_thresholdr*   r+   r,   daskconfiggetr   r-   
isinstancer   r&   callabler   r   local_directorytuple
ValueErrorr4   sysmaxsizer   ospathjoin	TypeErrorr   r/   r.   r   r	   rA   periodic_callbacksr   r0   )selfr2   r3   r(   r&   r*   r+   r,   r-   funckwargsr<   pcs                9lib/python3.11/site-packages/distributed/worker_memory.py__init__zWorkerMemoryManager.__init__f   s{    /(=
 
 
 '7.$"'
 '
#
 &6-#!&
 &
"
 &6-#!&
 &
" KOO$IJJ	"+u"4"4+i:P:PdN++ #	DIId^^ !	t566 #JDQQ D!788		GNN DFF		e$$ 	LD&D>> 9 !7888t566 + D  +1+AEK 		 !DNN6NN		 	'	+/+E	 * %%2Pd6PR 
 #V3Y??.  DII DI$)^44 	VTTTUUU9 	QOPPP'6KOOHII(
 (
 (
$ $6eEEEEE 	=&e33)66/;;;! +V44,t3	 B ;=F%&67(>>>    returnNonec                   K   |j                                         }|                     ||           |                     ||           d{V  dS )a9  Track this process's memory usage and act accordingly.
        If process memory rises above the spill threshold (70%), start dumping data to
        disk until it goes below the target threshold (60%).
        If process memory rises above the pause threshold (80%), stop execution of new
        tasks.
        N)monitorget_process_memory_maybe_pause_or_unpause_maybe_spill)rU   r2   memorys      rY   rA   z"WorkerMemoryManager.memory_monitor   s_       2244$$VV444///////////r[   rc   c                   | j         du rd S | j        sJ || j        z  }|| j         k    r| j                                         |j        t
          j        k    rht                              dt          |dz            t          |          | j        t          | j                  nd           t
          j        |_        d S d S |j        t
          j        k    rht                              dt          |dz            t          |          | j        t          | j                  nd           t
          j        |_        d S d S )NFz^Worker is at %d%% memory usage. Pausing worker.  Process memory: %s -- Worker memory limit: %sd   r]   z^Worker is at %d%% memory usage. Resuming worker. Process memory: %s -- Worker memory limit: %s)r,   r(   r0   collectstatusr   runningrD   warningr4   r   paused)rU   r2   rc   fracs       rY   ra   z+WorkerMemoryManager._maybe_pause_or_unpause   sL   %..F    ))$,,,&&(((}..%%Ds
OO (((4 !!2333   !' /. ]fm++!!@D3JV$$$0 T.///   #NFMMM ,+r[   c                b   K    j         du rdS t           j        d          rt           j        d          sdS  j        sJ  j        z  }| j         k    rdS t                              d|dz             dfdd fd}t          j         |            d           d{V  dS )z|If process memory is above the ``spill`` threshold, evict keys until it goes
        below the ``target`` threshold
        FNfastevictz>Worker is at %.0f%% memory usage. Start spilling data to disk.re   labelr   valuer.   unitstrr\   r]   c                p    t          | t                    s| f}                     dg| |R |           d S )Nzmemory-monitor)rI   rL   digest_metric)ro   rp   rq   r2   s      rY   metrics_callbackz:WorkerMemoryManager._maybe_spill.<locals>.metrics_callback  sI    eU++ !  "2!AU!AD!A!A5IIIIIr[   c                    K   t          j                  5                                  d {V  d d d            d S # 1 swxY w Y   d S N)r   add_callback_spill)rc   ru   rU   r2   s   rY   _z+WorkerMemoryManager._maybe_spill.<locals>._  s      +,<== 2 2 kk&&1111111112 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2s   AA	Azmemory-monitor-spillname)ro   r   rp   r.   rq   rr   r\   r]   )r\   r]   )r+   hasattrr&   r(   rD   debugasynciocreate_task)rU   r2   rc   rk   rz   ru   s   ```  @rY   rb   z WorkerMemoryManager._maybe_spill   s&      %..F ty&)) 	G1L1L 	F    ))4---FL3J	
 	
 	

	J 	J 	J 	J 	J 	J	2 	2 	2 	2 	2 	2 	2 	2 	2 !!!##,BCCCCCCCCCCCCr[   c                l  K   | j         sJ d}| j         | j        p| j        z  }d}||z
  }t                      x}}t	          t
          | j                  }	||k    r+|	j        s<t          	                    dt          |          t          | j                              n|	                                }
|
dk    rn||
z  }|dz  }|j                                        }||k    r8||k    r2| j                                         |j                                        }t                      }||z
  | j        k    r|                     ||           |}||z
  dk    r(t%          j        d           d{V  t                      }||k    +|r+t                              d|t          |                     dS dS )zGEvict keys until the process memory goes below the ``target`` thresholdr   a  Unmanaged memory use is high. This may indicate a memory leak or the memory may not be released to the OS; see https://distributed.dask.org/en/latest/worker-memory.html#memory-not-released-back-to-the-os for more information. -- Unmanaged memory: %s -- Worker memory limit: %s   g      ?NzMoved %d tasks worth %s to disk)r(   r*   r+   r   r   r   r&   rm   rD   ri   r   rn   r_   r`   r0   rf   r/   ra   r   sleepr~   )rU   r2   rc   total_spilledr<   countneedlast_checked_for_pauselast_yieldedr&   weightnows               rY   ry   zWorkerMemoryManager._spill"  s          "'E4+E
 09;$di00voo9 
%%I
 !(( !233   ZZ\\F|| V#MQJE^6688Ft## "**,,,::<<++C ++d.JJJ,,VV<<<),& \!C''mA&&&&&&&&&({{[ voo^  	1]++    	 	r[   rB   )excluder   Container[str]dictc                   d | j                                         D             }t                              | j                  |d<   |S )Nc                D    i | ]\  }}|                     d           ||S )rz   )
startswith).0kvs      rY   
<dictcomp>z0WorkerMemoryManager._to_dict.<locals>.<dictcomp>k  s/    PPPAall3>O>OP1PPPr[   r&   )__dict__itemsr   fromkeysr&   )rU   r   infos      rY   _to_dictzWorkerMemoryManager._to_dictj  s?    PP!4!4!6!6PPP}}TY//Vr[   )r2   r"   r3   r4   r(   r5   r&   r#   r*   r6   r+   r6   r,   r6   )r2   r"   r\   r]   )r2   r"   rc   r4   r\   r]   )r   r   r\   r   )__name__
__module____qualname____doc____annotations__rZ   r   rA   ra   rb   ry   r   rB   r[   rY   r%   r%   J   sD         $ &%%%222211111111####"""" %+ %)@D?C?Ca? a? a? a? a? a?F 0 0 0 Z0+ + + +B&D &D &D &DRF F F FP 57        r[   r%   c                  N    e Zd ZU ded<   ded<   ded<   ded<   d	d
ddZddZdS )NannyMemoryManagerr'   r(   r)   memory_terminate_fractionzfloat | Noner/   r4   _last_terminated_pidr1   )r(   nannyr!   r5   c                  t          ||j        t                    | _        t          j                            d          | _        t          t          j                            d          d          | _	        t          | j	        t          t          f          sJ d| _        | j        r@| j        dur9t          t          | j        |          | j	        dz            }||j        d<   d S d S d S )	Nr8   z#distributed.worker.memory.terminater=   Fr>   r   r@   rA   )rC   r3   nanny_loggerr(   rF   rG   rH   r   r   r/   rI   r4   r.   r   r   r	   rA   rT   )rU   r   r(   rX   s       rY   rZ   zNannyMemoryManager.__init__v  s     /%.
 
 
 *.1*
 *
& (7KOOHII(
 (
 (
$ $6eEEEEE$&! 	<!?u!L!L!+U33,t3 B :<E$%5666	< 	<!L!Lr[   r\   r]   c           	        |j         t          j        k    s$|j        |j        j        |j        j        j        dS |j        j        }	 t          j        |j                                                  j        }n(# t          t
          j
        t
          j        f$ r Y dS w xY w|| j        z  | j        k    rdS | j        |j        k    rYt                              d|j         d|j         d| j        dz  dd           |j        | _        |                                 dS t                              d|j         d|j         dt&          rd	nd
           |                                 dS )zCTrack worker's memory. Restart if it goes above terminate fraction.NzWorker z (pid=z) exceeded re   z.0fz% memory budget. Restarting...z) is slow to %szterminate; trying againzaccept SIGTERM; sending SIGKILL)rg   r   rh   processpidpsutilProcessmemory_inforssProcessLookupErrorNoSuchProcessAccessDeniedr(   r   r   r   ri   worker_address	terminater   kill)rU   r   r   rc   s       rY   rA   z!NannyMemoryManager.memory_monitor  s    LFN**}$}$,}$(0F-'	^GK00<<>>BFF"F$8&:MN 	 	 	FF	 D%%)GGGF$33   %.    gk    1C7N       
 )0D%    R%.RRgkRRR 7))6   LLNNNNNs   	0A: :!BBN)r   r!   r(   r5   )r   r!   r\   r]   )r   r   r   r   rZ   rA   rB   r[   rY   r   r   p  s}         5555)))) %+	< < < < < <41 1 1 1 1 1r[   r   r(   str | float | Noner3   r4   total_coresr9   logging.Loggerr\   r'   c                  | d S | }| dk    r-t          t          j        t          d||z            z            } t	          t
          t                    5  t          |           } t          | t                    r"| dk    rt          | t          j        z            } d d d            n# 1 swxY w Y   t          | t                    rt          |           } nt          |           } t          | t                     sJ | dk    rd S t          j        | k     r:|                    d|t          t          j                             t          j        S | S )Nr1   r   r   zBIgnoring provided memory limit %s due to system memory limit of %s)r4   r   MEMORY_LIMITminr   rM   rS   r.   rI   rr   r   ri   r   )r(   r3   r   r9   origs        rY   rC   rC     s    tDv6.Q;8N1O1OOPP	*i	(	( C C\**lE** 	C|q/@/@|f.AABBLC C C C C C C C C C C C C C C
 ,$$ )"<00<((lC(((((qt\))P,--	
 	
 	

 ""s   AB''B+.B+
config_keyrr   deprecated_param_namedeprecated_param_valuer6   r)   c                    |#t          j        d| d|  dt                     |S t          j                            |           S )Nz
Parameter zY has been deprecated and will be removed in a future version; please use dask config key z instead)warningswarnFutureWarningrF   rG   rH   )r   r   r   s      rY   rE   rE     sc    
 ).  GQ   		
 	
 	
 &%;??:&&&r[   wNanny | Workerr|   r]   c           
         t          j        dt          |           j         d| dt          |           j         d| t                     d S )NzThe `.z` attribute has been moved to `z.memory_manager.)r   r   typer   r   )r   r|   s     rY   _warn_deprecatedr     sl    M	5Q  	5 	54 	5 	5GG	5 	5.2	5 	5    r[   c                  2    e Zd ZU ded<   ddZddZddZdS ) DeprecatedMemoryManagerAttributerr   r|   ownerr   r\   r]   c                    || _         d S rw   r{   )rU   r   r|   s      rY   __set_name__z-DeprecatedMemoryManagerAttribute.__set_name__  s    			r[   instanceNanny | Worker | Noner   c                h    |d S t          || j                   t          |j        | j                  S rw   )r   r|   getattrmemory_managerrU   r   r   s      rY   __get__z(DeprecatedMemoryManagerAttribute.__get__  s4    449---x.	:::r[   r   rp   c                f    t          || j                   t          |j        | j        |           d S rw   )r   r|   setattrr   )rU   r   rp   s      rY   __set__z(DeprecatedMemoryManagerAttribute.__set__  s1    49---'E:::::r[   N)r   r   r|   rr   r\   r]   r   r   r   r   r\   r   )r   r   rp   r   r\   r]   )r   r   r   r   r   r   r   rB   r[   rY   r   r      s[         III   ; ; ; ;; ; ; ; ; ;r[   r   c                      e Zd Zd	dZdS )
DeprecatedMemoryMonitorr   r   r   r   r\   r   c                ^    |d S t          |d           t          |j        j        |          S )NrA   )r   r	   r   rA   r   s      rY   r   zDeprecatedMemoryMonitor.__get__  s4    4#3444x.=xHHHr[   Nr   )r   r   r   r   rB   r[   rY   r   r     s.        I I I I I Ir[   r   )
r(   r   r3   r4   r   r4   r9   r   r\   r'   )r   rr   r   rr   r   r6   r\   r)   )r   r   r|   rr   r\   r]   )Lr   
__future__r   r   loggingrP   rN   r   collections.abcr   r   r   r   
contextlibr   	functoolsr	   typingr
   r   r   r   r   r   dask.configrF   dask.systemr   dask.typingr   
dask.utilsr   r   r   distributedr   distributed.compatibilityr   r   distributed.corer   distributed.metricsr   r   distributed.spillr   r   distributed.utilsr   r   r   distributed.utils_perfr   typing_extensionsr    distributed.nannyr!   distributed.workerr"   objectrr   rL   r   r#   r   	getLoggerrD   	addFilterr   r%   r   rC   rE   r   r   r   rB   r[   rY   <module>r      s    ( # " " " " "   				 



  I I I I I I I I I I I I             ; ; ; ; ; ; ; ; ; ; ; ; ; ;      ! ! ! ! ! !       A A A A A A A A A A       ? ? ? ? ? ? ? ? # # # # # # 8 8 8 8 8 8 8 8 ; ; ; ; ; ; ; ; D D D D D D D D D D . . . . . . *++++++ ('''''))))))!&3;RV,,-cUN3;//0	(3sF{33
4d38n
DE
"      "!"=>>   ))*IJJ K K K w !;<<c c c c c c c cL	Q Q Q Q Q Q Q Qn !! ! ! ! !H' ' ' '    ; ; ; ; ; ; ; ;$I I I I I I I I I Ir[   