
    >ieh/                    &   d dl mZ d dlmZ d dlmZ d dlmZmZm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZ  ed	d
          Z G d de          Z edd          Z G d de          Z G d de          Z edd          Z G d de          ZdS )    )annotations)
namedtuple)starmap)PipeProcesscurrent_process)sleep)default_timer)Callback)import_requiredTaskData)keytask
start_timeend_time	worker_idc                  \     e Zd ZdZd Z fdZ fdZd Zd Zd Z	d Z
d	 Zd
 Zd Z xZS )Profilera  A profiler for dask execution at the task level.

    Records the following information for each task:
        1. Key
        2. Task
        3. Start time in seconds since the epoch
        4. Finish time in seconds since the epoch
        5. Worker id

    Examples
    --------

    >>> from operator import add, mul
    >>> from dask.threaded import get
    >>> from dask.diagnostics import Profiler
    >>> dsk = {'x': 1, 'y': (add, 'x', 10), 'z': (mul, 'y', 2)}
    >>> with Profiler() as prof:
    ...     get(dsk, 'z')
    22

    >>> prof.results        # doctest: +SKIP
    [TaskData(key='y', task=(add, 'x', 10), start_time=..., end_time=..., worker_id=...),
     TaskData(key='z', task=(mul, 'y', 2), start_time=..., end_time=..., worker_id=...)]

    These results can be visualized in a bokeh plot using the ``visualize``
    method. Note that this requires bokeh to be installed.

    >>> prof.visualize()    # doctest: +SKIP

    You can activate the profiler globally

    >>> prof.register()

    If you use the profiler globally you will need to clear out old results
    manually.

    >>> prof.clear()
    >>> prof.unregister()

    c                L    i | _         g | _        i | _        d | _        d | _        d S N)_resultsresults_dskr   r   selfs    8lib/python3.11/site-packages/dask/diagnostics/profile.py__init__zProfiler.__init__<   s)    	    c                    |                                   t                      | _        t                                                      S r   clearr
   r   super	__enter__r   	__class__s    r   r#   zProfiler.__enter__C   1    

'//ww  """r   c                V    t                      | _         t                      j        | S r   r
   r   r"   __exit__r   argsr%   s     r   r)   zProfiler.__exit__H   "    %uww&&r   c                :    | j                             |           d S r   r   updater   dsks     r   _startzProfiler._startL       	r   c                H    t                      }|||         |f| j        |<   d S r   r
   r   )r   r   r1   statestarts        r   _pretaskzProfiler._pretaskO   s(    !3s8U3cr   c                P    t                      }| j        |xx         ||fz  cc<   d S r   r5   )r   r   valuer1   r6   idends          r   	_posttaskzProfiler._posttaskS   s3    oocsBi'r   c           	        d | j                                         D             }| xj        t          t	          t
          |                                                    z  c_        | j                                          d S )Nc                @    i | ]\  }}t          |          d k    ||S )   )len).0kvs      r   
<dictcomp>z$Profiler._finish.<locals>.<dictcomp>X   s)    IIIDAqSVVq[[1a[[[r   )r   itemsr   listr   r   valuesr!   )r   r1   r6   failedr   s        r   _finishzProfiler._finishW   si    IIDM$7$7$9$9IIIWXw~~/?/?@@AAAr   c                N    ddl m}  || j        | j        | j        | j        fi |S )Nr   )
plot_tasks)"dask.diagnostics.profile_visualizerL   r   r   r   r   )r   kwargsrL   s      r   _plotzProfiler._plot\   sF    AAAAAAzL$)T_dm
 
GM
 
 	
r   c                     ddl m}  || fi |S zVisualize the profiling run in a bokeh plot.

        See also
        --------
        dask.diagnostics.profile_visualize.visualize
        r   )	visualizerM   rR   r   rN   rR   s      r   rR   zProfiler.visualizec   /     	A@@@@@y(((((r   c                v    | j                                          | j        dd= i | _        d| _        d| _        dS z#Clear out old results from profilerN)r   r!   r   r   r   r   r   s    r   r!   zProfiler.clearn   s<    LO	r   )__name__
__module____qualname____doc__r   r#   r)   r2   r8   r=   rJ   rO   rR   r!   __classcell__r%   s   @r   r   r      s        ' 'R  # # # # #
' ' ' ' '  4 4 4( ( (  

 
 
	) 	) 	)      r   r   ResourceData)timememcpuc                  n     e Zd ZdZddZd Zd Zd Z fdZ fdZ	d	 Z
d
 Zd ZeZd Zd Zd Z xZS )ResourceProfilera   A profiler for resource use.

    Records the following each timestep
        1. Time in seconds since the epoch
        2. Memory usage in MB
        3. % CPU usage

    Examples
    --------

    >>> from operator import add, mul
    >>> from dask.threaded import get
    >>> dsk = {'x': 1, 'y': (add, 'x', 10), 'z': (mul, 'y', 2)}
    >>> with ResourceProfiler() as prof:
    ...     get(dsk, 'z')
    22

    These results can be visualized in a bokeh plot using the ``visualize``
    method. Note that this requires bokeh to be installed.

    >>> prof.visualize() # doctest: +SKIP

    You can activate the profiler globally

    >>> prof.register()

    If you use the profiler globally you will need to clear out old results
    manually.

    >>> prof.clear()

    Note that when used as a context manager data will be collected throughout
    the duration of the enclosed block. In contrast, when registered globally
    data will only be collected while a dask scheduler is active.

    >>> prof.unregister()
       c                Z    || _         d| _        d | _        g | _        d | _        d | _        d S NF)_dt_entered_trackerr   r   r   )r   dts     r   r   zResourceProfiler.__init__   s0    r   c                F    | j         d uo| j                                         S r   )ri   is_aliver   s    r   _is_runningzResourceProfiler._is_running   s"    }D(ET]-C-C-E-EEr   c                    |                                  s2t          | j                  | _        | j                                         | j        j                            d           d S )Ncollect)rm   _Trackerrg   ri   r7   parent_connsendr   s    r   _start_collectzResourceProfiler._start_collect   sZ    !! 	"$TX..DMM!!!!&&y11111r   c                   |                                  rj| j        j                            d           | j                            t          t          | j        j                                                             d S d S )N	send_data)	rm   ri   rq   rr   r   extendr   r^   recvr   s    r   _stop_collectzResourceProfiler._stop_collect   ss     	YM%**;777Ldm6O6T6T6V6V W WXXXXX	Y 	Yr   c                    d| _         |                                  t                      | _        |                                  t                                                      S NT)rh   r!   r
   r   rs   r"   r#   r$   s    r   r#   zResourceProfiler.__enter__   sJ    

'//ww  """r   c                    d| _         |                                  |                                  t                      | _         t                      j        |  d S rf   )rh   rx   closer
   r   r"   r)   r*   s     r   r)   zResourceProfiler.__exit__   sN    

%$r   c                .    |                                   d S r   )rs   r0   s     r   r2   zResourceProfiler._start   s    r   c                @    | j         s|                                  d S d S r   )rh   rx   )r   r1   r6   rI   s       r   rJ   zResourceProfiler._finish   s.    } 	!     	! 	!r   c                r    |                                  r"| j                                         d| _        dS dS )z%Shutdown the resource tracker processN)rm   ri   shutdownr   s    r   r|   zResourceProfiler.close   s?     	!M""$$$ DMMM	! 	!r   c                0    g | _         d | _        d | _        d S r   )r   r   r   r   s    r   r!   zResourceProfiler.clear   s    r   c                B    ddl m}  || j        | j        | j        fi |S )Nr   )plot_resources)rM   r   r   r   r   )r   rN   r   s      r   rO   zResourceProfiler._plot   s6    EEEEEE~dlDOT]UUfUUUr   c                     ddl m}  || fi |S rQ   rS   rT   s      r   rR   zResourceProfiler.visualize   rU   r   rd   )rX   rY   rZ   r[   r   rm   rs   rx   r#   r)   r2   rJ   r|   __del__r!   rO   rR   r\   r]   s   @r   rc   rc   z   s       $ $L   F F F2 2 2Y Y Y
# # # # #           ! ! !! ! ! G  
V V V
	) 	) 	) 	) 	) 	) 	)r   rc   c                  6     e Zd ZdZd fd	Zd Zd Zd Z xZS )rp   z.Background process for tracking resource usagerd   c                    t                                                       d| _        || _        t	                      j        | _        t                      \  | _        | _	        d S rz   )
r"   r   daemonrj   r   pid
parent_pidr   rq   
child_conn)r   rj   r%   s     r   r   z_Tracker.__init__   sN    )++/,0FF)$///r   c                    | j         j        s3| j                             d           | j                                          |                                  d S )Nr   )rq   closedrr   r|   joinr   s    r   r   z_Tracker.shutdown   sM    & 	%!!*---""$$$		r   c                `    | j         gfd| j                                         D             z   S )Nc                Z    g | ]'}|j         k    |                                d k    %|(S )zombie)r   status)rB   pr   s     r   
<listcomp>z)_Tracker._update_pids.<locals>.<listcomp>   s=      
  
  
#!((**PXBXBXABXBXBXr   )parentchildren)r   r   s    `r   _update_pidsz_Tracker._update_pids   sH    }  
  
  
  
{++-- 
  
  
 
 	
r   c                &   t          dd          }|                    | j                  | _        t	                      }g }	 	 | j                                        }n# t          $ r Y (w xY w|dk    rn|dk    r|                     |          }|r| j        	                                st                      }dx}}|D ]J}		 |	                                j        }
|	                                }||
z  }||z  };# t          $ r Y Gw xY w|                    ||dz  |f           t!          | j                   || j        	                                n"|dk    r| j                            |           g }7| j                                         d S )	Npsutilz9Tracking resource usage requires `psutil` to be installedTr   ro   r   g    .Aru   )r   r   r   r   r   r   rw   KeyboardInterruptr   pollr
   memory_inforsscpu_percent	Exceptionappendr	   rj   rr   r|   )r   r   r   datamsgpsticr`   ra   r   mem2cpu2s               r   runz_Tracker.run   s    Q
 
 nnT_55	o**,,$   j  	!!&&s++ #do&:&:&<&< #'//C !MC# 	( 	((#$==??#6D#$==??D
  4KC4KCC  ) ! ! ! D! KKcCi 5666$'NNN  #do&:&:&<&< # ##$$T***5	6 	s$   A 
A)(A)-C99
DDr   )	rX   rY   rZ   r[   r   r   r   r   r\   r]   s   @r   rp   rp      sp        883 3 3 3 3 3  
 
 

#  #  #  #  #  #  # r   rp   	CacheData)r   r   metric
cache_time	free_timec                  X     e Zd ZdZddZ fdZ fdZd Zd Zd Z	d	 Z
d
 Zd Z xZS )CacheProfilera]  A profiler for dask execution at the scheduler cache level.

    Records the following information for each task:
        1. Key
        2. Task
        3. Size metric
        4. Cache entry time in seconds since the epoch
        5. Cache exit time in seconds since the epoch

    Examples
    --------

    >>> from operator import add, mul
    >>> from dask.threaded import get
    >>> from dask.diagnostics import CacheProfiler
    >>> dsk = {'x': 1, 'y': (add, 'x', 10), 'z': (mul, 'y', 2)}
    >>> with CacheProfiler() as prof:
    ...     get(dsk, 'z')
    22

    >>> prof.results    # doctest: +SKIP
    [CacheData(key='y', task=(add, 'x', 10), metric=1, cache_time=..., free_time=...),
     CacheData(key='z', task=(mul, 'y', 2), metric=1, cache_time=..., free_time=...)]

    The default is to count each task (``metric`` is 1 for all tasks). Other
    functions may used as a metric instead through the ``metric`` keyword. For
    example, the ``nbytes`` function found in ``cachey`` can be used to measure
    the number of bytes in the cache.

    >>> from cachey import nbytes                   # doctest: +SKIP
    >>> with CacheProfiler(metric=nbytes) as prof:  # doctest: +SKIP
    ...     get(dsk, 'z')
    22

    The profiling results can be visualized in a bokeh plot using the
    ``visualize`` method. Note that this requires bokeh to be installed.

    >>> prof.visualize() # doctest: +SKIP

    You can activate the profiler globally

    >>> prof.register()

    If you use the profiler globally you will need to clear out old results
    manually.

    >>> prof.clear()
    >>> prof.unregister()

    Nc                    |                                   |r|nd | _        |r	|| _        d S |r|j        | _        d S d| _        d S )Nc                    dS )Nrd    )r:   s    r   <lambda>z(CacheProfiler.__init__.<locals>.<lambda>`  s    1 r   count)r!   _metric_metric_namerX   )r   r   metric_names      r   r   zCacheProfiler.__init__^  s]    

!'<vv__ 	( +D 	( &D 'Dr   c                    |                                   t                      | _        t                                                      S r   r    r$   s    r   r#   zCacheProfiler.__enter__h  r&   r   c                V    t                      | _         t                      j        | S r   r(   r*   s     r   r)   zCacheProfiler.__exit__m  r,   r   c                :    | j                             |           d S r   r.   r0   s     r   r2   zCacheProfiler._startq  r3   r   c           
     F   t                      }|                     |          |f| j        |<   |d         | j                                        z  D ]P}| j                            |          \  }}	| j                            t          |||         ||	|                     Qd S )Nreleased)r
   r   _cachekeyspopr   r   r   )
r   r   r:   r1   r6   r;   trC   r   r7   s
             r   r=   zCacheProfiler._posttaskt  s    OO LL//3Cz"T[%5%5%7%77 	H 	HA KOOA..MFEL	!SVVUA F FGGGG	H 	Hr   c           
         t                      }| j                                        D ]9\  }\  }}| j                            t          |||         |||                     :| j                                         d S r   )r
   r   rF   r   r   r   r!   )r   r1   r6   rI   r   rC   r   r7   s           r   rJ   zCacheProfiler._finish{  sz    OO"&+"3"3"5"5 	H 	HAL	!SVVUA F FGGGGr   c                Z    ddl m}  || j        | j        | j        | j        | j        fi |S )Nr   )
plot_cache)rM   r   r   r   r   r   r   )r   rN   r   s      r   rO   zCacheProfiler._plot  sQ    AAAAAAzLIOM
 
 
 
 	
r   c                     ddl m}  || fi |S rQ   rS   rT   s      r   rR   zCacheProfiler.visualize  rU   r   c                L    g | _         i | _        i | _        d| _        d| _        dS rW   )r   r   r   r   r   r   s    r   r!   zCacheProfiler.clear  s)    	r   )NN)rX   rY   rZ   r[   r   r#   r)   r2   r=   rJ   rO   rR   r!   r\   r]   s   @r   r   r   *  s        1 1f( ( ( (# # # # #
' ' ' ' '  H H H  

 

 

	) 	) 	)      r   r   N)
__future__r   collectionsr   	itertoolsr   multiprocessingr   r   r   r_   r	   timeitr
   dask.callbacksr   
dask.utilsr   r   r   r^   rc   rp   r   r   r   r   r   <module>r      s   " " " " " " " " " " " "       : : : : : : : : : :                   # # # # # # & & & & & & :F 
b b b b bx b b bJ z.*@AAm) m) m) m) m)x m) m) m)`8  8  8  8  8 w 8  8  8 v JE 	
t t t t tH t t t t tr   