
    0FieP6                    ,   d dl mZ d dlZd dlZd dlZd dlmZ d dlmZ d dl	m
Z
 d dlmZmZ d dlmZ d dlmZ d d	lmZ d d
lmZ  ej        e          ZddZ G d de          Z G d de          Zd Z G d de          Z G d de          ZdS )    )annotationsN)defaultdict)default_timer)ClassVar)groupbyvalmap)tokenize)	key_split)SchedulerPlugin)timeFc                X   t                      }t                      }t          |           }|rz|                                }|j        }||v r"|s|j        r,|j        |                    |           |sK|                    |           |                    |j                   |z||fS )z
    All keys that need to compute for these keys to finish.

    If *complete* is false, omit tasks that are busy processing or
    have finished executing.
    )	setlistpopkeywho_has	exceptionaddextenddependencies)taskscompleteouterrorsstacktsr   s          @lib/python3.11/site-packages/distributed/diagnostics/progress.pydependent_keysr      s     %%CUUFKKE
 &YY[[f#:: 	BJ 	<#JJsOOO R_%%%  & ;    c                  4    e Zd ZdZddZd Zd Zd	 Zdd
ZdS )Progressa  Tracks progress of a set of keys or futures

    On creation we provide a set of keys or futures that interest us as well as
    a scheduler.  We traverse through the scheduler's dependencies to find all
    relevant keys on which our keys depend.  We then plug into the scheduler to
    learn when our keys become available in memory at which point we record
    their completion.

    State
    -----
    keys: set
        Set of keys that are not yet computed
    all_keys: set
        Set of all keys that we track

    This class performs no visualization.  However it is used by other classes,
    notably TextProgressBar and ProgressWidget, which do perform visualization.
    r   皙?FNc                "   |pdt          ||||           | _        d |D             | _        d | j        D             | _        || _        || _        || _        || _        d| _        t                      | _	        d| _
        d | _        i | _        d S )Nz	progress-c                @    h | ]}t          |d           r|j        n|S )r   )hasattrr   .0ks     r   	<setcomp>z$Progress.__init__.<locals>.<setcomp>F   s-    EEE1ga//6QUUQEEEr   c                    h | ]}|S  r+   r&   s     r   r)   z$Progress.__init__.<locals>.<setcomp>G   s    ***1Q***r   r   F)r	   namekeys	schedulerr   _minimum_dtlast_durationr   _start_time_runningstatusextra)selfr-   r.   minimumdtr   r,   s          r   __init__zProgress.__init__D   s    OOwH(M(MOO	EEEEE	**	***	" (??


r   c                   K    j         }|                     j        j                  s9t	          j        d           d {V  |                     j        j                  9 fd|D             }d  _          j                                        t          | j                  \   _	        } j        s j	        
                                 _         nt          |d          \   _         } j	                            |            xj         | j	        z  z  c_          j         s                     d d            t                              d           |D ]}                     |d ddd	
           d S )N皙?c                4    g | ]}j         j        |         S r+   r.   r   r'   r(   r6   s     r   
<listcomp>z"Progress.setup.<locals>.<listcomp>X   #    777Q%a(777r   r   Fr   r   Set up Progress keyserredzprogress-setupTstimulus_idr   )r-   issubsetr.   r   asynciosleep
add_pluginr   r   all_keyscopyupdatestoploggerdebug
transition)r6   r-   r   r   _r(   s   `     r   setupzProgress.setupR   s     y-- 455 	&-%%%%%%%%% -- 455 	& 8777$777	!!$''' .ut} M M Mv} 	A**,,DII)%%@@@LDIqT"""		Vdm++		y 	0II$I///+,,, 	 	AOO4.>$     	 	r   c                   || j         v r\|dk    rV|dk    rPt                              d|           | j                             |           | j         s|                                  || j        v r=|dk    r7t                              d           |                     |d         |           || j         v r:|dk    r6t                              d	|           |                     d
|           d S d S d S )N
processingmemoryzProgress sees key %srD   Progress sees task erredr   rB   	forgotten,A task was cancelled (%s), stopping progressT)r-   rO   rP   removerN   rK   )r6   r   startfinishargskwargss         r   rQ   zProgress.transitiono   s    $) 5 5&H:L:LLL/555IS!!!9 		$-Fg$5$5LL3444II{ 3I===$)+ 5 5LLGMMMII#I.....  5 5r   c                .    |                                   d S N)rN   r6   r.   s     r   restartzProgress.restart   s    		r   c                &   | j         | j        j        v r | j                            | j                    |r:d| _        | j                            | j        j        |         j        |d           nd| _        t          
                    d           d S )N)r,   errorrB   finishedzRemove Progress plugin)r,   r.   pluginsremove_pluginr4   r5   rM   r   r   rO   rP   )r6   r   r   s      r   rN   zProgress.stop   s    9...N((di(888 	%!DKJ"n237A#NN    %DK-.....r   )r   r"   FN)NN)	__name__
__module____qualname____doc__r9   rS   rQ   rb   rN   r+   r   r   r!   r!   0   ss         &     :/ / /   
/ 
/ 
/ 
/ 
/ 
/r   r!   c                  @     e Zd ZdZ	 ddddddd fdZd	 Zd
 Z xZS )MultiProgressa  Progress variant that keeps track of different groups of keys

    See Progress for most details.

    Parameters
    ----------

    func : Callable (deprecated)
        Function that splits keys. This defaults to ``key_split`` which
        aligns with naming conventions chosen in the dask project (tuples,
        hyphens, etc..)

    group_by : Callable | Literal["spans"] | Literal["prefix"], default: "prefix"
        How to group keys to display multiple bars. Defaults to "prefix",
        which uses ``key_split`` from dask project

    State
    -----
    keys: dict
        Maps group name to set of not-yet-complete keys for that group
    all_keys: dict
        Maps group name to set of all keys for that group

    Examples
    --------
    >>> split = lambda s: s.split('-')[0]
    >>> p = MultiProgress(['y-2'], func=split)  # doctest: +SKIP
    >>> p.keys   # doctest: +SKIP
    {'x': {'x-1', 'x-2', 'x-3'},
     'y': {'y-1', 'y-2'}}
    Nprefixr   r"   F)funcgroup_byr7   r8   r   c                   |t          j        dt                     |}|dv rt          n|| _        d | _        dt          |||||           }t                                          ||||||           d S )Nz$`func` is deprecated, use `group_by`)category)Nrn   zmulti-progress-)r7   r8   r   r,   )	warningswarnDeprecationWarningr
   rp   ro   r	   superr9   )
r6   r-   r.   ro   rp   r7   r8   r   r,   	__class__s
            r   r9   zMultiProgress.__init__   s     M6AS    H%-1A%A%A		x	R$'2x!P!PRR)WhT 	 	
 	
 	
 	
 	
r   c                   	K    j         }|                     j        j                  s9t	          j        d           d {V  |                     j        j                  9 fd|D             }d  _          j                                        t          | j                  \   _	        } j        s j	        
                                 _         nt          |d          \   _         } j	                            |            xj         | j	        z  z  c_          j         s                     d d             j        dk    rC j        j        d         }|r|j        nd 	 	fdfd j	        D             }|j         _        n$ j        d	k    rt$           _        n j         _        t'          t(          t+           j         j                              _         t'          t(          t+           j         j	                             _	         j	        D ]!}| j         vrt)                       j         |<   "|D ]}                     |d d
dd           t.                              d           d S )Nr;   c                4    g | ]}j         j        |         S r+   r=   r>   s     r   r?   z'MultiProgress.setup.<locals>.<listcomp>   r@   r   rA   FrB   spansc                    j         j        |          j        j        }r d                    |         j                  n|}||fS )Nz, )r.   r   groupspan_idjoinr,   )r(   r}   	span_namer6   	span_defss      r   	group_keyz&MultiProgress.setup.<locals>.group_key   sG    ..q17?BKXDIIi&8&=>>>QX	 '))r   c                (    i | ]}| |          S r+   r+   )r'   r(   r   s     r   
<dictcomp>z'MultiProgress.setup.<locals>.<dictcomp>   s#    AAAa!YYq\\AAAr   rn   rD   zmultiprogress-setupTrE   rC   )r-   rG   r.   r   rH   rI   rJ   r   r   rK   rL   rM   rN   rp   
extensionsrz   getro   r
   r   r   r   rQ   rO   rP   )
r6   r-   r   r   rR   	spans_ext
group_keysr(   r   r   s
   `       @@r   rS   zMultiProgress.setup   s     y-- 455 	&-%%%%%%%%% -- 455 	& 8777$777	!!$''' .ut} M M Mv} 	A**,,DII)%%@@@LDIqT"""		Vdm++		y 	0II$I///=G##1':I+4>	$I* * * * * *
 BAAA4=AAAJ"DII]h&&!DIIDI 3	49 = =>>	sGDIt}$E$EFF 	% 	%A	!!"uu	! 	 	AOO4.Ct      	+,,,,,r   c                   |dk    r|dk    r| j                             |                     |          d           }|r||v r|                    |           | j         r&t	          | j                                                   s|                                  |dk    rqt                              d           |                     |          }|| j	        v r9|| j	        |         v r*|                     |                    d          |           |dk    r`|                     |          }|| j	        v rD|| j	        |         v r7t                              d|           |                     d	
           d S d S d S d S )NrU   rV   rD   rW   r   rB   rX   rY   T)r   )
r-   r   ro   rZ   anyvaluesrN   rO   rP   rK   )r6   r   r[   r\   r]   r^   sr(   s           r   rQ   zMultiProgress.transition   sq   L  Vx%7%7	diinnd33A SAXX9 C	(8(8(:(:$;$; 		WLL3444		#ADM!!cT]1-=&=&=		FJJ{$;$;	EEE[  		#ADM!!cT]1-=&=&=KSQQQ		D	)))))	 ! !!&=&=r   r`   )rh   ri   rj   rk   r9   rS   rQ   __classcell__)rw   s   @r   rm   rm      s         F 

 
 
 
 
 
 
 
.1- 1- 1-f* * * * * * *r   rm   c                    t          | d          \  }}t          |d          \  }}|r|dd|dd|ddS |r
|dd|ddS |ddS )zFormat seconds into a human readable form.

    >>> format_time(10.4)
    '10.4s'
    >>> format_time(1000.4)
    '16min 40.4s'
    >>> format_time(100000.4)
    '27hr 46min 40.4s'
    <   z2.0fzhr zmin z4.1fr   )divmod)tmr   hs       r   format_timer     s     !R==DAq!R==DAq 222Q22222222	
 '''a'''''||||r   c                  (    e Zd ZdZdZd Zd Zd ZdS )AllProgressz,Keep track of all keys, grouped by key_splitzall-progressc                   t          t                    | _        t          t                    | _        t          d           | _        || _        | j        j                                        D ]}|j	        }|j
        j        }| j        |                             |           | j        |j                 |                             |           |j        dk    r| j        |xx         |j        z  cc<   |                    |            d S )Nc                 *    t          t                    S r`   )r   r   r+   r   r   <lambda>z&AllProgress.__init__.<locals>.<lambda>,  s    S)9)9 r   r   )r   r   allintnbytesstater.   r   r   r   rn   r,   r   rJ   )r6   r.   r   r   rn   s        r   r9   zAllProgress.__init__)  s    s##!#&& !9!9::
".&--// 	1 	1B&CY^FHV  %%%Jrx (,,S111yA~~F###ry0###T"""""r   c                   | j         j        |         }|j        j        }| j        |                             |           	 | j        |         |                             |           n# t          $ r Y nw xY w|dk    r%|j	        dk    r| j	        |xx         |j	        z  cc<   |dk    r%|j	        dk    r| j	        |xx         |j	        z  cc<   |dk    r(| j        |         |                             |           d S | j        |         }|                    |           |sU| j        |= | j	        
                    |d            | j                                        D ]}	|	
                    |d            d S d S )NrV   r   rX   )r.   r   rn   r,   r   r   r   rZ   KeyErrorr   r   r   )
r6   r   r[   r\   r]   r^   r   rn   r   vs
             r   rQ   zAllProgress.transition9  s   ^!#&S!!!	Juf%,,S1111 	 	 	D	 HaK29,X")q..K29,[  Jvv&**3///// AHHSMMM (HV$---**,, ( (AEE&$''''	( (( (s    &A' '
A43A4c                j    | j                                          | j                                         d S r`   )r   clearr   ra   s     r   rb   zAllProgress.restartS  s.    
r   N)rh   ri   rj   rk   r,   r9   rQ   rb   r+   r   r   r   r   $  sL        66D# # # ( ( (4    r   r   c                  Z    e Zd ZU dZdZded<   ded<   ded<   ded	<   d
 ZddZd Zd Z	dS )GroupTimingzCKeep track of high-level timing information for task group progresszgroup-timingzClassVar[str]r,   zlist[float]r   zdict[str, list[float]]computenthreadsc                J    || _         d| _        |                                  d S )Ng      ?)r.   r8   _initra   s     r   r9   zGroupTiming.__init__`  s%    "  	

r   returnNonec                p    t                      }|gdz  | _         i | _        | j        j        gdz  | _        dS )z7Shared initializatoin code between __init__ and restart   N)r   r   r.   total_nthreadsr   )r6   nows     r   r   zGroupTiming._initi  s9    ff EAI	67!;r   c                   |dk    rK|dk    rF|                     d          }|s t                              d| d           d S t                      }|| j        d<   | j        d         | j        d         z
  | j        k    r| j        d         | j        z   | j        d<   | j                            |           | j                            | j        j                   | j	        
                                D ]}|                    d           | j        d         | j        d         z
  | j        k    | j        j        |         }	|	j        }
|
j        | j	        vr%dgt          | j                  z  | j	        |
j        <   |D ]}|d	         d
k    r|d         }|d         }t          | j                  dz
  }|dk    r3| j        |dz
           |k    r|dz  }|dk    r| j        |dz
           |k    |dk    r]||k    rW|t          | j        |dz
           |          z
  }| j	        |
j                 |xx         |z  cc<   ||z  }|dz  }|dk    r||k    Wd S d S d S )NrU   rV   
startstopszTask z@ finished processing, but timing information seems to be missingg        actionr   rN   r[      r   )r   rO   warningr   r8   appendr   r.   r   r   r   r   r|   r,   lenmax)r6   r   r[   r\   r]   r^   r   r   gtaskr|   	startstoprN   idxdeltas                  r   rQ   zGroupTiming.transitionu  s    L  Vx%7%7L11J !C ! ! !    &&CDIbM)B-$)B-/$'99 $	" 7	"	  %%%$$T^%BCCC,,.. " "AHHSMMMM )B-$)B-/$'99 >',DJE z--,/53ty>>+AUZ('  	X&)33 (!'*$)nnq( Agg$)C!G"4t";";1HC Agg$)C!G"4t";"; Agg$,, 3tyq'95#A#AAEL,S111U:111EMD1HC Agg$,,O ! %7%78 r   c                .    |                                   d S r`   )r   ra   s     r   rb   zGroupTiming.restart  s    

r   N)r   r   )
rh   ri   rj   rk   r,   __annotations__r9   r   rQ   rb   r+   r   r   r   r   X  s         MM(D((((####  
< 
< 
< 
<0 0 0d    r   r   )F)
__future__r   rH   loggingrs   collectionsr   timeitr   typingr   tlzr   r   	dask.baser	   
dask.utilsr
   distributed.diagnostics.pluginr   distributed.metricsr   	getLoggerrh   rO   r   r!   rm   r   r   r   r+   r   r   <module>r      s   " " " " " "    # # # # # #                                             : : : : : : $ $ $ $ $ $		8	$	$   6\/ \/ \/ \/ \/ \/ \/ \/~~* ~* ~* ~* ~*H ~* ~* ~*B  (1 1 1 1 1/ 1 1 1hP P P P P/ P P P P Pr   