
    0Fie                     F   d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlm	Z	m
Z
 d dlmZ d dlmZ d dlmZmZ d dlZd dlmZ d dlmZ d d	lmZ erd d
lmZ  ej        e          Zd Zd$dZd%d&dZ d'dZ! G d d          Z" G d de
e"                   Z#ddd(d#Z$dS ))    )annotationsN)IterableSequence)import_module)
ModuleType)TYPE_CHECKINGcast)tmpfile)Server)import_file)Clientc                   |s| j                             dd          s|S |r| j                             dd          sgd |D             }|D ]}t          j        |          t          j        dt          |          dk    rdnddd	                    |          d
          d | j                             d          D             }d |                                D             }t          |          dk    r6t          j        dt          |	                                          z            |r|st          j        d          |s|S |d         }t          j
        |d          } |                    | t          |                     |S )zKClick option callback providing validation of preload subcommand arguments.preloadNc                <    g | ]}|                     d           |S )-
startswith).0vs     6lib/python3.11/site-packages/distributed/preloading.py
<listcomp>z)validate_preload_argv.<locals>.<listcomp>$   s)    AAAq||C/@/@A1AAA    zGot unexpected extra argument   s z: ( )c                L    i | ]!}t          |          |t          |          "S  )is_webaddress_import_module)r   names     r   
<dictcomp>z)validate_preload_argv.<locals>.<dictcomp>,   s@       T""nT""  r   c           	         g | ]<}t          t          |d d          t          j                  +t          |d d          =S )
dask_setupN)
isinstancegetattrclickCommand)r   ms     r   r   z)validate_preload_argv.<locals>.<listcomp>2   sR       gat44emDD<&&  r   z<Multiple --preload modules with click-configurable setup: %szPUnknown argument specified: %r Was click-configurable --preload target provided?r   Fallow_extra_args)paramsgetr(   NoSuchOption
UsageErrorlenjoinvalueslistkeysContext
parse_args)ctxparamvalueunexpected_argsapreload_modulespreload_commandspreload_commands           r   validate_preload_argvr@      s    	488  
SZ^^It44 
AAeAAA  	( 	(A$Q'''%jj1nnss",,chhuoooo?
 
 	

 JNN9--  O  ''))   q  J?''))**+
 
 	

  
% 
^
 
 	
  .*1-
-%
@
@
@CsDKK000Lr   r   strreturnboolc                ,    |                      d          S )N)zhttp://zhttps://r   )r   s    r   r    r    M   s    <</000r   r"   file_dir
str | Noner   c                ^   |                      d          r|t          j                            |           }t          j                            ||          }t          j                            |          r0t          j        | |          st          	                    d|           t          j        | |           t          |          d         }nt          |           d         }nd| vr0| t          j        vrt          |            t          j        |          }nwt!          d          5 }t#          |d          5 }|                    |            ddd           n# 1 swxY w Y   t'          ||	          cddd           S # 1 swxY w Y   t                              d
|            |S )a  Imports module and extract preload interface functions.

    Import modules specified by name and extract 'dask_setup'
    and 'dask_teardown' if present.

    Parameters
    ----------
    name : str
        Module name, file path, or text of module or script
    file_dir : string
        Path of a directory where files should be copied

    Returns
    -------
    Nest dict of names to extracted module interface components if present
    in imported module.
    z.pyNzFile name collision: %sr   r   )	extensionw)moderE   zImport preload module: %s)endswithospathbasenamer2   existsfilecmpcmploggererrorshutilcopyr   sysmodulesr   r
   openwriter!   info)r"   rE   rO   copy_dstmodulefnfs          r   r!   r!   Q   s   $ }}U 9w''--Hw||Hh77Hw~~h'' F{422 FLL!:HEEEKh''' **1-FF &&q)FF	Ds{""$T" u%%% 	9bs### q              !"x888	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9
 KK+T222Ms6   +F=EFE#	#F&E#	'FFFurlc           	        t                               d|            t          |           sJ dd l}|                                5 }|                    d| |j                            g dd                    }|j        }d d d            n# 1 swxY w Y   t          || d          }t          |           }t          ||j                   |S )	NzDownloading preload at %sr   GET)i  i  i  i  g?)status_forcelistbackoff_factor)methodr`   retriesexec)rS   r[   r    urllib3PoolManagerrequestutilRetrydatacompiler   rg   __dict__)r`   rh   httpresponsesourcecompiledr]   s          r   _download_modulert      s   
KK+S111 NNN					 
$<<L&&!5!5!5" '     
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 vsF++H__F6?###Ms   <BBBc                  Z    e Zd ZU dZded<   ded<   ded<   ded	<   d
ed<   ddZd Zd ZdS )Preloada  
    Manage state for setup/teardown of a preload module

    Parameters
    ----------
    dask_server: dask.distributed.Server
        The Worker or Scheduler
    name: str
        module name, file name, or web address to load
    argv: [str]
        List of string arguments passed to click-configurable `dask_setup`.
    file_dir: str
        Path of a directory where files should be copied
    Server | Clientdask_objectrA   r"   	list[str]argvrF   rE   r   r]   Iterable[str]c                   || _         || _        t          |          | _        || _        t
                              d| j                   t          |          rt          |          | _	        d S t          ||          | _	        d S )NzCreating preload: %s)rx   r"   r4   rz   rE   rS   r[   r    rt   r]   r!   )selfrx   r"   rz   rE   s        r   __init__zPreload.__init__   sv     '	JJ	 *DI666 	9*400DKKK(x88DKKKr   c                  K   t          | j        dd          }|rt                              d| j                   t          |t          j                  r]|                    d| j	        d          } |j
        | j        g|j        R i |j        }t          j        |          r
| d{V  dS dS  || j                  }t          j        |          r| d{V  dS dS dS )z-Run when the server finishes its start methodr%   NzRun preload setup: %sFr+   )r'   r]   rS   r[   r"   r&   r(   r)   make_contextrz   callbackrx   argsr-   inspectisawaitable)r}   r%   contextresultfutures        r   startzPreload.start   s-     T[,==
 	!KK/;;;*em44 !$11 $)e 2   -,$'.|  7>~  &v.. ! LLLLLLLLL! ! $D$455&v.. ! LLLLLLLLL	! 	!! !r   c                   K   t          | j        dd          }|rNt                              d| j                    || j                  }t          j        |          r| d{V  dS dS dS )z+Run when the server starts its close methoddask_teardownNzRun preload teardown: %s)r'   r]   rS   r[   r"   rx   r   r   )r}   r   r   s      r   teardownzPreload.teardown   s      _dCC 	KK2DI>>>"]4#344F"6** 		 	 r   N)rx   rw   r"   rA   rz   r{   rE   rF   )__name__
__module____qualname____doc____annotations__r~   r   r   r   r   r   rv   rv      s           !   IIIOOO9 9 9 9&! ! !(    r   rv   c                  @    e Zd ZU ded<   ddZddZddZd	 ZddZdS )PreloadManagerlist[Preload]	_preloadspreloadsc                    || _         d S Nr   )r}   r   s     r   r~   zPreloadManager.__init__   s    !r   rB   Nonec                   K   | j         D ]M}	 |                                 d {V  # t          $ r# t                              d|j                   Y Jw xY wd S )NzFailed to start preload: %s)r   r   	ExceptionrS   	exceptionr"   r}   r   s     r   r   zPreloadManager.start   s      ~ 	N 	NGNmmoo%%%%%%%% N N N  !>MMMMMN	N 	Ns   (*AAc                   K   t          | j                  D ]M}	 |                                 d {V  # t          $ r# t                              d|j                   Y Jw xY wd S )NzFailed to tear down preload: %s)reversedr   r   r   rS   r   r"   r   s     r   r   zPreloadManager.teardown   s      // 	R 	RGR&&(((((((((( R R R  !BGLQQQQQR	R 	Rs   5*A"!A"c                    | j         |         S r   r   )r}   indexs     r   __getitem__zPreloadManager.__getitem__   s    ~e$$r   intc                *    t          | j                  S r   )r1   r   )r}   s    r   __len__zPreloadManager.__len__   s    4>"""r   N)r   r   )rB   r   )rB   r   )	r   r   r   r   r~   r   r   r   r   r   r   r   r   r      s         " " " "N N N NR R R R% % %# # # # # #r   r   rK   dask_serverrw   r   str | list[str]preload_argvlist[str] | list[list[str]]c                   t          |t                    r|g}|r=t          |d         t                    r"t          d|          gt          |          z  }n#|s!t          dg           gt          |          z  }t          |          t          |          k    r/t	          dt          |           dt          |                     t           fdt          ||          D                       S )Nr   ry   z1preload and preload_argv have mismatched lengths z != c                :    g | ]\  }}t          ||          S r   )rv   )r   prz   r   rE   s      r   r   z$process_preloads.<locals>.<listcomp>  s;     	
 	
 	
4 KD(33	
 	
 	
r   )r&   rA   r	   r1   
ValueErrorr   zip)r   r   r   rE   s   `  `r   process_preloadsr      s!    '3 ) >
<?C88 >[,7783w<<G >["--.W=
7||s<((((57||5 5!$\!2!25 5
 
 	

 	
 	
 	
 	
 	
w55	
 	
 	
  r   )r   rA   rB   rC   r   )r"   rA   rE   rF   rB   r   )r`   rA   rB   r   )
r   rw   r   r   r   r   rE   rF   rB   r   )%
__future__r   rQ   r   loggingrM   rU   rW   collections.abcr   r   	importlibr   typesr   typingr   r	   r(   
dask.utilsr
   distributed.corer   distributed.utilsr   distributed.clientr   	getLoggerr   rS   r@   r    r!   rt   rv   r   r   r   r   r   <module>r      s   " " " " " "    				  



 . . . . . . . . # # # # # #       & & & & & & & &        # # # # # # ) ) ) ) ) ) *))))))		8	$	$. . .b1 1 1 1- - - - -`   2D D D D D D D DN# # # # #Xg& # # #@         r   