
    0Fie|]                       d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	m
Z
mZ d dlmZ d dlmZ d dlmZmZmZmZ d dlmZ d dlmZ d dlZd d	lmZmZ d d
lmZ d dlmZm Z  d dl!m"Z" d dl#m$Z$ d dl%m&Z& d dl'm(Z( d dl)m*Z*m+Z+m,Z,m-Z- erd dl.m/Z/ d dl0m1Z1m2Z2  ej3        e4          Z5 G d d          Z6 ed          Z7d$dZ8 G d de$          Z9d%d!Z:d%d"Z;ej<        d#             Z=dS )&    )annotationsN)	Awaitable	Generator)suppress)isawaitable)TYPE_CHECKINGAnyClassVarTypeVar)gen)IOLoop)parse_bytesparse_timedelta)get_template)Statusrpc)Adaptive)Cluster)	Scheduler)Security)NoOpAwaitableTimeoutErrorimport_termsilence_logging_cmgr)Self)NannyWorkerc                      e Zd ZdZed             Zej        d             ZddZd Zd Z	d Z
d	 Zd
 Zd Zd Zd ZdS )ProcessInterfacea)  
    An interface for Scheduler and Worker processes for use in SpecCluster

    This interface is responsible to submit a worker or scheduler process to a
    resource manager like Kubernetes, Yarn, or SLURM/PBS/SGE/...
    It should implement the methods below, like ``start`` and ``close``
    c                    | j         S N)_statusselfs    7lib/python3.11/site-packages/distributed/deploy/spec.pystatuszProcessInterface.status5   s
    |    c                b    t          |t                    st          d|          || _        d S )NzExpected Status; got )
isinstancer   	TypeErrorr"   )r$   
new_statuss     r%   r&   zProcessInterface.status9   s7    *f-- 	DBJBBCCC!r'   Nc                    t          | dd           | _        d | _        t          j                    | _        t          j        | _        t          j	                    | _
        d S )Naddress)getattrr-   external_addressasyncioLocklockr   createdr&   Event_event_finished)r$   	schedulernames      r%   __init__zProcessInterface.__init__?   sG    tY55 $LNN	n&}r'   c                F      fd} |                                             S )Nc                    K    j         4 d {V   j        t          j        k    r1                                  d {V   j        t          j        k    sJ 	 d d d           d {V  n# 1 d {V swxY w Y    S r!   )r2   r&   r   r3   startrunningr#   s   r%   _z%ProcessInterface.__await__.<locals>._G   s     y 9 9 9 9 9 9 9 9;&.00**,,&&&&&&&;&.888889 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 Ks   AA++
A58A5	__await__r$   r=   s   ` r%   r?   zProcessInterface.__await__F   s2    	 	 	 	 	 qss}}r'   c                ,   K   t           j        | _        dS )a  Submit the process to the resource manager

        For workers this doesn't have to wait until the process actually starts,
        but can return once the resource manager has the request, and will work
        to make the job exist in the future

        For the scheduler we will expect the scheduler's ``.address`` attribute
        to be available after this completes.
        N)r   r<   r&   r#   s    r%   r;   zProcessInterface.startP   s       nr'   c                ^   K   t           j        | _        | j                                         dS )a8  Close the process

        This will be called by the Cluster object when we scale down a node,
        but only after we ask the Scheduler to close the worker gracefully.
        This method should kill the process a bit more forcefully and does not
        need to worry about shutting down gracefully
        N)r   closedr&   r5   setr#   s    r%   closezProcessInterface.close\   s,       m  """""r'   c                H   K   | j                                          d{V  dS )z"Wait until the server has finishedN)r5   waitr#   s    r%   finishedzProcessInterface.finishedg   s3      "'')))))))))))r'   c                |    dt           j                            t          |                      d| j        j         dS )N<z	: status=>)daskutilstypenametyper&   r7   r#   s    r%   __repr__zProcessInterface.__repr__k   s6    P4:&&tDzz22PPT[=MPPPPr'   c                H    t          d                              |           S )Nzprocess_interface.html.j2)process_interface)r   renderr#   s    r%   _repr_html_zProcessInterface._repr_html_n   s#    788??RV?WWWr'   c                   K   |  d {V  | S r!    r#   s    r%   
__aenter__zProcessInterface.__aenter__q   s      






r'   c                >   K   |                                   d {V  d S r!   )rE   )r$   exc_type	exc_value	tracebacks       r%   	__aexit__zProcessInterface.__aexit__u   s,      jjllr'   )NN)__name__
__module____qualname____doc__propertyr&   setterr8   r?   r;   rE   rH   rP   rT   rW   r\   rV   r'   r%   r   r   ,   s            X ]" " ]"
/ / / /  
% 
% 
%	# 	# 	#* * *Q Q QX X X      r'   r   _TawAwaitable[_T]returnc                   K   |  d {V S r!   rV   )rd   s    r%   _wrap_awaitablerh   |   s      888888Or'   c                  X    e Zd ZU dZ ej                    Zded<   	 	 	 	 	 	 	 	 	 	 d6 fd	Zd7d8 fdZ	 fdZ
d Zd9dZ fdZd:dZ fdZd Zd;dZd;dZd<dZd Zd Zed              Zd! ZeZed"             Zed#             Zedej        ddddfd= fd1Ze d>d5            Z! xZ"S )?SpecClustera  Cluster that requires a full specification of workers

    The SpecCluster class expects a full specification of the Scheduler and
    Workers to use.  It removes any handling of user inputs (like threads vs
    processes, number of cores, and so on) and any handling of cluster resource
    managers (like pods, jobs, and so on).  Instead, it expects this
    information to be passed in scheduler and worker specifications.  This
    class does handle all of the logic around asynchronously cleanly setting up
    and tearing things down at the right times.  Hopefully it can form a base
    for other more user-centric classes.

    Parameters
    ----------
    workers: dict
        A dictionary mapping names to worker classes and their specifications
        See example below
    scheduler: dict, optional
        A similar mapping for a scheduler
    worker: dict
        A specification of a single worker.
        This is used for any new workers that are created.
    asynchronous: bool
        If this is intended to be used directly within an event loop with
        async/await
    silence_logs: bool
        Whether or not we should silence logging when setting up the cluster.
    name: str, optional
        A name to use when printing out the cluster, defaults to type name

    Examples
    --------
    To create a SpecCluster you specify how to set up a Scheduler and Workers

    >>> from dask.distributed import Scheduler, Worker, Nanny
    >>> scheduler = {'cls': Scheduler, 'options': {"dashboard_address": ':8787'}}
    >>> workers = {
    ...     'my-worker': {"cls": Worker, "options": {"nthreads": 1}},
    ...     'my-nanny': {"cls": Nanny, "options": {"nthreads": 2}},
    ... }
    >>> cluster = SpecCluster(scheduler=scheduler, workers=workers)

    The worker spec is stored as the ``.worker_spec`` attribute

    >>> cluster.worker_spec
    {
       'my-worker': {"cls": Worker, "options": {"nthreads": 1}},
       'my-nanny': {"cls": Nanny, "options": {"nthreads": 2}},
    }

    While the instantiation of this spec is stored in the ``.workers``
    attribute

    >>> cluster.workers
    {
        'my-worker': <Worker ...>
        'my-nanny': <Nanny ...>
    }

    Should the spec change, we can await the cluster or call the
    ``._correct_state`` method to align the actual state to the specified
    state.

    We can also ``.scale(...)`` the cluster, which adds new workers of a given
    form.

    >>> worker = {'cls': Worker, 'options': {}}
    >>> cluster = SpecCluster(scheduler=scheduler, worker=worker)
    >>> cluster.worker_spec
    {}

    >>> cluster.scale(3)
    >>> cluster.worker_spec
    {
        0: {'cls': Worker, 'options': {}},
        1: {'cls': Worker, 'options': {}},
        2: {'cls': Worker, 'options': {}},
    }

    Note that above we are using the standard ``Worker`` and ``Nanny`` classes,
    however in practice other classes could be used that handle resource
    management like ``KubernetesPod`` or ``SLURMJob``.  The spec does not need
    to conform to the expectations of the standard Dask Worker class.  It just
    needs to be called with the provided options, support ``__await__`` and
    ``close`` methods and the ``worker_address`` property..

    Also note that uniformity of the specification is not required.  Other API
    could be added externally (in subclasses) that adds workers of different
    specifications into the same dictionary.

    If a single entry in the spec will generate multiple dask workers then
    please provide a `"group"` element to the spec, that includes the suffixes
    that will be added to each name (this should be handled by your worker
    class).

    >>> cluster.worker_spec
    {
        0: {"cls": MultiWorker, "options": {"processes": 3}, "group": ["-0", "-1", -2"]}
        1: {"cls": MultiWorker, "options": {"processes": 2}, "group": ["-0", "-1"]}
    }

    These suffixes should correspond to the names used by the workers when
    they deploy.

    >>> [ws.name for ws in cluster.scheduler.workers.values()]
    ["0-0", "0-1", "0-2", "1-0", "1-1"]
    z&ClassVar[weakref.WeakSet[SpecCluster]]
_instancesNFT   c                B   ||rt          j                    }t          j                    x| _        }t          j                    | _        t          j        |          | _	        t          j        |          pi | _
        t          j        |          | _        d | _        i | _        d| _        |pt                      | _        t#                      | _        |rG|                    t)          |                     |                    t)          |d                     | j                            |            d | _        |pt1          |           j        | _        |	| _        t9                                          ||||
           | j        s| j                                          | !                    | j"                   	 | !                    | j#                   d S # tH          $ r5 | !                    | j%                   | j        &                                  w xY wd S )Nr   )levelbokeh)rn   root)asynchronousloopr7   scheduler_sync_interval)'r   current
contextlib	ExitStack_SpecCluster__exit_stackweakrefWeakSet_createdcopyscheduler_specworker_specnew_specr6   workers_ir   securityrD   _futuresenter_contextr   rk   add_correct_state_waitingrO   r]   _nameshutdown_on_closesuperr8   called_from_running_loop_loop_runnerr;   sync_start_correct_state	ExceptionrE   stop)r$   r   r6   workerrq   rr   r   silence_logsr7   r   rs   stack	__class__s               r%   r8   zSpecCluster.__init__   s    <L<>##D$.$8$:$::E))"i	229W--3	&)) .HJJ 	X 4< H H HIII 4<g V V VWWWD!!!&*#0T$ZZ0
!2%$;	 	 	
 	
 	
 , 	##%%%IIdk"""		$-.....   		$*%%%!&&(((	 	s   G ?Htimeoutfloat | Nonerf   Awaitable[None] | Nonec                    t                                          |          }| j        s| j                                         |S r!   )r   rE   rq   r   r   )r$   r   rd   r   s      r%   rE   zSpecCluster.close$  s=    WW]]7##  	%""$$$	r'   c                &  K   | j         t          j        k    r/t          j        d           d {V  | j         t          j        k    /| j         t          j        k    rd S | j         t          j        k    rt          d          t          j                    | _	        t          j        | _         | j
        )	 dd l}ddi}n# t          $ r Y nw xY wt          |d| _
        	 | j        i| j
        d         }t          |t                     rt#          |          } |di | j
                            di           | _        | j         d {V | _        t'          t)          | j        d	d           p| j        j        | j                            d
                    | _        t3                                                       d {V  d S # t6          $ rC}t          j        | _         |                                  d {V  t=          d|           |d }~ww xY w)Ng{Gz?zCluster is closedr   	dashboardT)clsoptionsr   r   r/   client)connection_argszCluster failed to start: rV   )r&   r   startingr0   sleepr<   rC   
ValueErrorr1   _lockr|   distributed.dashboardImportErrorr   r6   r)   strr   getr   r.   r-   r   get_connection_argsscheduler_commr   r   r   failed_closeRuntimeError)r$   distributedr   r   er   s        r%   r   zSpecCluster._start*  sH     kV_,,-%%%%%%%%% kV_,,;&.((F;&-''0111\^^
o&.,,,, '-     +4"H"HD	G~%)%0c3'' +%c**C!$!N!Nt':'>'>y"'M'M!N!N'+~!5!5!5!5!5!5"%(:DAA *>) $ A A( K K# # #D
 ''..""""""""""" 	G 	G 	G -DK++-->1>>??QF	Gs+   4B= =
C
	C
C$G 
H>HHc                |    | j         r| j         S t          j        |                                           }|| _         |S r!   )r   r0   ensure_future_correct_state_internal)r$   tasks     r%   r   zSpecCluster._correct_stateQ  s?    & 	..()E)E)G)GHHD*.D'Kr'   Nonec           	        K    j         4 d {V  d  _        t           j                  t           j                  z
  }|rj j        j        t          j        k    r. j	        
                    t          |                     d {V   fd|D             }t          j        |  d {V  |D ]}| j        v r j        |= t           j                  t           j                  z
  }g }|D ]} j        |         }|d         |                    di           }}d|vr|                                }||d<   t!          |t"                    rt%          |          } |t'           j        dd           p j        j        fi |}	 j                            |	           |                    |	           |rd |D             }
t          j        |
           d {V   j                            t5          t7          ||                               |D ]}t9          j                   |_        t          j        |
  d {V  d d d           d {V  d S # 1 d {V swxY w Y   d S )N)r   c                    g | ]<}|j         v t          j        j         |                                                   =S rV   )r   r0   create_taskrE   .0wr$   s     r%   
<listcomp>z7SpecCluster._correct_state_internal.<locals>.<listcomp>b  sM       DL(( 'Q(=(=(?(?@@(((r'   r   r   r7   contact_addressc                6    g | ]}t          j        |          S rV   )r0   r   r   r   s     r%   r   z7SpecCluster._correct_state_internal.<locals>.<listcomp>~  s#    IIIAw4Q77IIIr'   )r   r   rD   r   r}   r6   r&   r   r<   r   retire_workerslistr0   gatherr   r{   r)   r   r   r.   r-   rz   r   appendrG   updatedictziprx   ref_cluster)r$   to_closetasksr7   to_openr   dr   optsr   worker_futsr   s   `           r%   r   z#SpecCluster._correct_state_internalZ  s`     : +	3 +	3 +	3 +	3 +	3 +	3 +	3 +	3*.D'4<((3t/?+@+@@H ->(FN::-<<T(^^<TTTTTTTTT   %  
 ne,,,,,,,,  + +4<''T*$*++c$,.?.??GG ' '$T*eHaeeIr&:&:T%%99;;D#'DLc3'' +%c**CDN,=tDD .~-   
 !!&)))v&&&& 	3IIIIIl;/////////##DWg)>)>$?$?@@@  3 3A!(T!2!2AJJ nk22222222W+	3 +	3 +	3 +	3 +	3 +	3 +	3 +	3 +	3 +	3 +	3 +	3 +	3 +	3 +	3 +	3 +	3 +	3 +	3 +	3 +	3 +	3 +	3 +	3 +	3 +	3 +	3 +	3 +	3 +	3s   H6I
I$'I$c                D    |dk    rs j         d                  d          fd}t          t          j                            d                    }t          j                                        ||           t                      	                    |           d S )Nremover   r7   c                 V   j         v r j        d         vrt          fdj        d                                         D                       sWj                            t          j        j                                                                       j         = d S d S d S d S )Nr   c              3  0   K   | ]}|d          k    V  dS )r7   NrV   )r   r   r7   s     r%   	<genexpr>z?SpecCluster._update_worker_status.<locals>.f.<locals>.<genexpr>  s@           &	T)           r'   )	r   scheduler_infoanyvaluesr   r   r0   r   rE   )msgr7   r$   s   r%   fz,SpecCluster._update_worker_status.<locals>.f  s    DL((4#6y#AAA        !%!4Y!?!F!F!H!H        B M%%g&;DL<N<T<T<V<V&W&WXXXT*** )(AAAAr'   z&distributed.deploy.lost-worker-timeout)
r   r   rL   configr   r0   get_running_loop
call_laterr   _update_worker_status)r$   opr   r   delayr7   r   s   ` `  @r%   r   z!SpecCluster._update_worker_status  s    >>&y1#6v>D
+ 
+ 
+ 
+ 
+ 
+ 
+ $ HII E $&&11%;;;%%b#.....r'   r$   r   Generator[Any, Any, Self]c                H     d fd} |                                             S )Nrf   r   c                 <  K    j         t          j        k    r                                  d {V   j         d {V                                    d {V   j        r;t          j        d  j        	                                D                        d {V   S )Nc                P    g | ]#}t          j        t          |                    $S rV   )r0   r   rh   r   s     r%   r   z4SpecCluster.__await__.<locals>._.<locals>.<listcomp>  s;         +OA,>,>??  r'   )
r&   r   r3   r   r6   r   r   r0   rG   r   r#   s   r%   r=   z SpecCluster.__await__.<locals>._  s      {fn,,kkmm#######.       %%'''''''''| l !%!4!4!6!6           Kr'   )rf   r   r>   r@   s   ` r%   r?   zSpecCluster.__await__  s7    	 	 	 	 	 	 qss}}r'   c                R  K   | j         t          j        k    r/t          j        d           d {V  | j         t          j        k    /| j         t          j        k    rd S | j         t          j        k    s| j         t          j        k    rt          j        | _         t          t                    5  | j
                                         d d d            n# 1 swxY w Y   |                     d          }t          |          r| d {V  |                                  d {V  t          j        | j          d {V  | j        r| j        4 d {V  t          t&                    5  | j                                         d {V  d d d            n# 1 swxY w Y   | j                                         d {V  d d d           d {V  n# 1 d {V swxY w Y   nt,                              d           | j        r| j                                         d {V  | j        D ];}|j         t          j        t          j        t          j        hv sJ |j                     <| j                            d d d            t;                                                       d {V  d S )Ng?r   z"Cluster closed without starting up)r&   r   closingr0   r   rC   r<   r   r   AttributeError	_adaptiver   scaler   r   r   r   r   r   OSError	terminate	close_rpcloggerwarningr6   rE   rz   rw   __exit__r   r   )r$   r   r   r   s      r%   r   zSpecCluster._close  s     kV^++-$$$$$$$$$ kV^++;&-''F;&.((DK6=,H,H .DK .)) & &##%%%& & & & & & & & & & & & & & & 

1A1~~ %%'''''''''.$-00000000" E: : : : : : : : :!'** > >"1;;=========> > > > > > > > > > > > > > >-77999999999: : : : : : : : : : : : : : : : : : : : : : : : : : :
 CDDD~ -n**,,,,,,,,,]  xNMM$    8	    	""4t444ggnnsH   /CCCG( FGF	GF	"G
GGc                   K   	 |  d {V  |                                   d {V  | j        t          j        k    sJ | S # t          $ r |                                  d {V   w xY wr!   )r   r&   r   r<   r   rE   r#   s    r%   rW   zSpecCluster.__aenter__  s      	JJJJJJJ%%''''''''';&.0000K 	 	 	**,,	s	   :? &A%intc                    | j         st          d          dD ]C}t          t                    5  | j         d         |         cddd           c S # 1 swxY w Y   Dt	          d          )z7Return the number of threads per worker for new workersz4To scale by cores= you must specify cores per worker)nthreadsncoresthreadscoresr   Nunreachable)r~   r   r   KeyErrorr   r$   r7   s     r%   _threads_per_workerzSpecCluster._threads_per_worker  s    } 	USTTT> 	6 	6D(## 6 6}Y/56 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6=)))s   AA	A	c                    | j         st          d          dD ]P}t          t                    5  t	          | j         d         |                   cddd           c S # 1 swxY w Y   Qt          d          )z2Return the memory limit per worker for new workerszQto scale by memory= your worker definition must include a memory_limit definition)memory_limitmemoryr   NzVto use scale(memory=...) your worker definition must include a memory_limit definition)r~   r   r   r   r   r   s     r%   _memory_per_workerzSpecCluster._memory_per_worker  s    } 	*  
 / 	C 	CD(## C C"4=#;D#ABBC C C C C C C C C C C C C C C C C C C &
 
 	
s    AA"	%A"	r   c           
     p   |Qt          |t          t          j        t	          |          |                                 z                                }|Dt          |t          t          j        ||                                 z                                }t          | j                  |k    rt          | j                  d | j
        d                                         D             z
  }t          | j                  |k    r6|r4| j        |                                = t          | j                  |k    r|4t          | j                  |k    r1| j                                         t          | j                  |k    1| j        t          j        t          j        fvr\t          | j                  |k     rD| j                            |                                            t          | j                  |k     D| j                            | j                   | j        rt1                      S d S )Nc                    h | ]
}|d          S )r7   rV   )r   vs     r%   	<setcomp>z$SpecCluster.scale.<locals>.<setcomp>  s)     8 8 8&	8 8 8r'   r   )maxr   mathceilr   r   r   lenr}   rD   r   r   poppopitemr&   r   r   rC   r   new_worker_specrr   add_callbackr   rq   r   )r$   nr   r   not_yet_launcheds        r%   r   zSpecCluster.scale  s   As49[%8%84;R;R;T;T%TUUVVWWAAs49UT-E-E-G-G%GHHIIJJAt  1$$"4#344 8 8#'#6y#A#H#H#J#J8 8 8   d&''!++0@+$%5%9%9%;%;< d&''!++0@+ $"##a''$$&&& $"##a'' ;v~v}===d&''!++ ''(<(<(>(>??? d&''!++ 		t2333 	# ??"	# 	#r'   c                    |S )zReturns new worker name.

        This can be overridden in SpecCluster derived classes to customise the
        worker names.
        rV   )r$   worker_numbers     r%   _new_worker_namezSpecCluster._new_worker_name  s
     r'   c                    |                      | j                  }|| j        v r3| xj        dz  c_        |                      | j                  }|| j        v 3|| j        iS )zReturn name and spec for the next worker

        Returns
        -------
        d: dict mapping names to worker specs

        See Also
        --------
        scale
        rl   )r	  r   r}   r~   )r$   new_worker_names     r%   r  zSpecCluster.new_worker_spec  sj     //88!111GGqLGG"33DG<<O !111  //r'   c                *    t          | j                  S r!   )boolr~   r#   s    r%   _supports_scalingzSpecCluster._supports_scaling0  s    DM"""r'   c                2   K   t           fd|D                       sYi  j                                        D ]/\  }}d|v r!|d         D ]}|t          |          |z   <   *||<   0fd|D             }|D ]}| j        v r j        |=   d {V  d S )Nc              3  *   K   | ]}|j         v V  d S r!   )r}   r   s     r%   r   z)SpecCluster.scale_down.<locals>.<genexpr>6  s+      ::Q1((::::::r'   groupc                <    h | ]}                     ||          S rV   )r   )r   r   mappings     r%   r   z)SpecCluster.scale_down.<locals>.<setcomp>?  s'    :::Qw{{1a((:::r'   )allr}   itemsr   )r$   r   r7   specsuffixr   r  s   `     @r%   
scale_downzSpecCluster.scale_down4  s      ::::'::::: 		;G".4466 ) )
dd??"&w- ; ;6:D		F 233; %)GDMM::::':::G 	( 	(AD$$$$Q'








r'   c                    t                      }| j                                        D ]F\  }d|v r(|                    fd|d         D                        1|                               G|S )Nr  c                4    h | ]}t                    |z   S rV   r   r   r  r7   s     r%   r   z#SpecCluster.plan.<locals>.<setcomp>M  $    KKK6CII.KKKr'   )rD   r}   r  r   r   r$   outr  r7   s      @r%   planzSpecCluster.planH  s{    ee*0022 	 	JD$$

KKKKT']KKKLLLL
r'   c                    t                      }| j        D ]b	 | j                 }n# t          $ r Y w xY wd|v r(|                    fd|d         D                        M|                               c|S )Nr  c                4    h | ]}t                    |z   S rV   r  r  s     r%   r   z(SpecCluster.requested.<locals>.<setcomp>[  r  r'   )rD   r   r}   r   r   r   r  s      @r%   	requestedzSpecCluster.requestedR  s    eeL 	 	D'-   $

KKKKT']KKKLLLL
s   (
55r   type[Adaptive]minimumfloatmaximumminimum_cores
int | Nonemaximum_coresminimum_memory
str | Nonemaximum_memorykwargsr	   c                B   |9t          |pdt          j        ||                                 z                      }|Ft          |pdt          j        t	          |          |                                 z                      }|7t          |t          j        ||                                 z                      }|Dt          |t          j        t	          |          |                                 z                      } t                      j	        d|||d|S )a  Turn on adaptivity

        This scales Dask clusters automatically based on scheduler activity.

        Parameters
        ----------
        minimum : int
            Minimum number of workers
        maximum : int
            Maximum number of workers
        minimum_cores : int
            Minimum number of cores/threads to keep around in the cluster
        maximum_cores : int
            Maximum number of cores/threads to keep around in the cluster
        minimum_memory : str
            Minimum amount of memory to keep around in the cluster
            Expressed as a string like "100 GiB"
        maximum_memory : str
            Maximum amount of memory to keep around in the cluster
            Expressed as a string like "100 GiB"

        Examples
        --------
        >>> cluster.adapt(minimum=0, maximum_memory="100 GiB", interval='500ms')

        See Also
        --------
        dask.distributed.Adaptive : for more keyword arguments
        Nr   )r   r%  r'  rV   )
r   r   r   r   r   r   minfloorr   adapt)
r$   r   r%  r'  r(  r*  r+  r-  r.  r   s
            r%   r2  zSpecCluster.adapt`  s/   P $1di8P8P8R8R(RSS G %1	+n558O8O8Q8QQRR G $MD4L4L4N4N$NOO G %
;~669P9P9R9RRSS G
 uww} 
w
 
CI
 
 	
r'   r7   r   r   c                    t                      )zJCreate an instance of this class to represent an existing cluster by name.)NotImplementedError)r   r7   s     r%   	from_namezSpecCluster.from_name  s     "###r'   )
NNNFNNFNTrl   r!   )r   r   rf   r   )rf   r   )r$   r   rf   r   )rf   r   )r   NN)r   r$  r%  r&  r'  r&  r(  r)  r*  r)  r+  r,  r-  r,  r.  r	   rf   r   )r7   r   rf   r   )#r]   r^   r_   r`   rx   ry   rk   __annotations__r8   rE   r   r   r   r   r?   r   rW   r   r   r   r	  r  ra   r  r  scale_upr   r#  r   r   infr2  classmethodr5  __classcell__)r   s   @r%   rj   rj      sc        i iV :I9J9JJJJJJ  !4 4 4 4 4 4l      %G %G %G %G %GN  ,3 ,3 ,3 ,3\/ / / / /.   "% % % % %N  * * * *
 
 
 
"# # # #4  0 0 0$ # # X#  $ H  X   X $,$($(%)%)=
 =
 =
 =
 =
 =
 =
~ $ $ $ [$ $ $ $ $r'   rj   r  dict[str, Any]argsr	   dict[str, Worker | Nanny]c           	         i }|                                  D ]P\  }}|d         }t          |t                    rt          |          } ||i |                    di           ||<   Q|S )Nr   r   )r  r)   r   r   r   )r  r<  r   kr   r   s         r%   	init_specr@    sw    G

 5 51hc3 	#c""CS$4!%%"3"344

Nr'   c                t   K   t          | g|R  }|r&t          j        |                                  d {V  |S r!   )r@  r0   r   r   )r  r<  r   s      r%   run_specrB    sQ      $t$$$G 0ngnn..////////Nr'   c                 L   t          t          j                  D ]} t          | dd          rvt	          t
          j        t                    5  t          | dt          j                  t          j        k    r| 	                    d           d d d            n# 1 swxY w Y   d S )Nr   Fr&   
   )r   )
r   rj   rk   r.   r   r   r   r   rC   rE   )clusters    r%   close_clustersrF    s    .// . .7/77 	.#*L99 . .7Hfm<<MMMM"M---. . . . . . . . . . . . . . .. .s   A BB	B	)rd   re   rf   rc   )r  r;  r<  r	   rf   r=  )>
__future__r   r0   atexitru   r{   loggingr   rx   collections.abcr   r   r   inspectr   typingr   r	   r
   r   tornador   tornado.ioloopr   rL   
dask.utilsr   r   dask.widgetsr   distributed.corer   r   distributed.deploy.adaptiver   distributed.deploy.clusterr   distributed.schedulerr   distributed.securityr   distributed.utilsr   r   r   r   typing_extensionsr   r   r   r   	getLoggerr]   r   r   rc   rh   rj   r@  rB  registerrF  rV   r'   r%   <module>rZ     s   " " " " " "           0 0 0 0 0 0 0 0             8 8 8 8 8 8 8 8 8 8 8 8       ! ! ! ! ! !  3 3 3 3 3 3 3 3 % % % % % % ( ( ( ( ( ( ( ( 0 0 0 0 0 0 . . . . . . + + + + + + ) ) ) ) ) )             *&&&&&& *)))))))		8	$	$J J J J J J J JZ WT]]   b$ b$ b$ b$ b$' b$ b$ b$J       . . . . .r'   