o
    Nrf                     @  sL  d dl mZ d dlZedZd dlZd dlZd dlZd dlm	Z	 d dl
mZ d dlmZmZmZ d dlmZ d dlmZ d d	lmZmZmZmZmZmZmZ d dlZd dlmZ d d
lmZmZm Z  d dl!m"Z"m#Z# d dl$m%Z% d dl&m'Z' d dl(m)Z)m*Z* d dl+m,Z,m-Z- d dl.m/Z/m0Z0m1Z1m2Z2 d dl3m4Z4 de/ev re	eddZe	eddZej5j6ej7dkddZ8ej59dZ:dd Z;dd Z<edddd Z=d d! Z>d"d# Z?d$d% Z@ej5Ad&ddgd'd( ZBej5Ad)d*d*ggej5Ad+ddgd,d- ZCej5Ad.g d/ej5Ad0d1d2gd3d4 ZDd5d6 ZEd7d8 ZFe:eddd9d: ZGej5jHd;d< ZIe:ej5Ad=d>ejJd?ej5jKd@ddAdBejJdCej5jKd@ddAdBgej5jHdDdE ZLedddFdG ZMdHdI ZNdJdK ZOdLdM ZPdNdO ZQdPdQ ZRdRdS ZSedddTdU ZTdVdW ZUedddXdY ZVej5AdZg d[ej5Ad&g d\d]d^ ZWe:ej5AdZd_ejJd`ej5jXdBdaejJdbej5jYdcdddBgej5Ad&g d\ej5Adeddgdfdg ZZdhdi Z[edddjdk Z\edddldm Z]dndo Z^edddpdq Z_drds Z`edddtdu Zaedddvdw Zbej5Adxdye-fdze,fgedd{gdz d|d}d~ Zcedddd Zddd Zeej59dedddd Zfedddd Zgdd Zhdd Zidd Zjdd Zkej5Adg ddd Zlej5Adg ddd Zmej5jXej5Addejno gdd Zpeddig ddd Zqedddd ZrdS )    )annotationsNdistributed)partial)add)ClientSchedulerPluginWorkerPlugin)cleanup)client)clustercluster_fixturegen_clusterlooploop_in_threadpopenvarying)computedelayedpersist)compute_as_if_collectionget_scheduler)	Blockwise)Delayed)
futures_ofwait)ShuffleLayerSimpleShuffleLayer)get_named_argsget_scheduler_locktmpdirtmpfile)incshould_check_stateF)r"   win32zTThe teardown of distributed.utils_test.cluster_fixture fails on windows CI currentlyreasonzrignore:Running on a single-machine scheduler when a distributed client is active might lead to unexpected results.c                  C     ddl m}  d S )Nr   r   )dask.distributedr   r'    r)   d/var/www/html/software/conda/envs/catlas/lib/python3.10/site-packages/dask/tests/test_distributed.pytest_can_import_client>      r+   c                  C  r&   )Nr   dumps)Zdask.distributed.protocolr.   r-   r)   r)   r*   test_can_import_nested_thingsB   r,   r/   Tc           	        s   t td}t|\}t|I d H  |j|jv s!|j|jv s!J t td}t|d\}}t|I d H  |j|jv sC|j|jv sEJ d S d S )N   
   )r   r!   r   r   keydata)	csabxZx2yy2oner)   r)   r*   test_persistF   s   
$r<   c                 C  s  t dd }|d }|d } t|dd|gd| dfddg}t|d d ts)J t|d d d ts6J t|d d tsAJ d	g d
ddddgf}t| |ksUJ t||g| ddgdd}|d d |u skJ |d d |u suJ |d  dksJ |dd  ddgfksJ d S )Nr0         r6   r7      r   r6   r7      )r0   r>      )   r>   F)traverserC   )r   r   
isinstancer   r   )r4   r6   r7   resultZsolresr)   r)   r*   test_persist_nestedU   s    rH   c                 C  s   t d}t d}|dg di}| ||g}||}|j| |j||gdd |	 s?t
|jj|j ts?J t t |ddg}W d    d S 1 sWw   Y  d S )	Npandasdask.dataframer8   r0   r>      r   Zaxisr0   r>   )pytestimportorskip	DataFramescatterfrom_delayedutils	assert_eqr   concat_dask_expr_enabledrE   dasklayers_namer   raises	TypeError)r4   pddddffuturesddfr)   r)   r*   !test_futures_to_delayed_dataframeh   s   


 "ra   c                 C  sX   t d}t d}|dtdi}|j|dd}|| }|jj||| d d S )NrI   rJ   r8      r>   npartitions	scheduler)	rN   rO   rP   rangefrom_pandasrR   
to_delayedrS   rT   )r4   r\   r]   r^   r`   r)   r)   r*   test_from_delayed_dataframez   s   

rj   fusec                 C  s   t d}t d}d}|t|t|d}|t|t|d}|j||dd }|j|ddd }|d7 }|d7 }tjd	|i |j|d
gddd}	|		  |	
 d
}
W d    n1 sfw   Y  |jj|
|j|d
gddd
dd d S )NrI   rJ   #   r8   r9   )r8   zrc   r1   r=   optimization.fuse.activer8   lefttasks)onhowshuffle_method)rr   rs   FZcheck_index)rN   rO   rP   rg   rh   rW   configsetmergeheadr   sort_valuesrS   rT   )r4   rk   r\   r]   sizeZdf1Zdf2Zddf1ddf2ddfmdfmr)   r)   r*   $test_fused_blockwise_dataframe_merge   s"   


r   rr   r6   	broadcastc                 C  s   t d}t d}|ddgd tdd}|ddgtdd}|j|dd}|j|dd}|j||||d	d
}	|	 }
|jj|
	d|j|||d	ddd d S )NrI   rJ   r0   r>   r@   )r6   Zb_left)r6   Zb_rightrc   rq   )rr   r   rt   r6   )rr   Fru   )
rN   rO   rP   rg   rh   rx   r   rS   rT   rz   )r4   rr   r   r\   r]   ZpdflZpdfrZdflZdfrr}   r~   r)   r)   r*   test_dataframe_broadcast_merge   s   


r   computation)Nr   dask.computezscheduler, use_distributed)NT)syncFc                   s   t dt dG dd dt}| |   fdd}| |}| du s.J | dd	 }|r?|d
ksAJ |dS dS )zvShould a collection use its default scheduler or the distributed
    scheduler when being computed within a task?
    rI   rJ   c                   @  s   e Zd Zdd Zdd ZdS )z<test_default_scheduler_on_worker.<locals>.UpdateGraphCounterc                   s   d|_ d S )Nr   Z_update_graph_count)selfrf   r)   r)   r*   start   s   
zBtest_default_scheduler_on_worker.<locals>.UpdateGraphCounter.startc                 _  s   | j d7  _ d S Nr0   r   )r   rf   argskwargsr)   r)   r*   update_graph   s   zItest_default_scheduler_on_worker.<locals>.UpdateGraphCounter.update_graphN)__name__
__module____qualname__r   r   r)   r)   r)   r*   UpdateGraphCounter   s    r   c                    s   d}  t| t| d}j|dd} d u r!|jd d	S  dkr.tj|d d	S  dkrAt|j|jt|jd d	S J )
Nr1   rm   r>   rc   re   r   r   FT)rP   rg   rh   r   rW   r   	__class__list)r{   r^   r`   r   r]   r\   rf   r)   r*   foo   s   
z-test_default_scheduler_on_worker.<locals>.fooTc                 S  s   | j S Nr   Zdask_schedulerr)   r)   r*   <lambda>   s    z2test_default_scheduler_on_worker.<locals>.<lambda>r>   r0   N)rN   rO   r   register_pluginsubmitrF   run_on_scheduler)r4   r   Zuse_distributedrf   r   r   rG   Znum_update_graphsr)   r   r*    test_default_scheduler_on_worker   s   


r   c                 C  s8   g d}|  ||g}t|}t||| ksJ d S )NrK   )rQ   dbrR   r   )r4   Lr_   r7   r)   r)   r*   test_futures_to_delayed_bag   s   
r   c                   s   t d t d ddlm} t d}|d| g} j fdd|D dd}|| |jgdd d S )	Nnumpy
dask.arrayr   rT   r=   c                   s    g | ]} j |jjd qS ))shapedtype)rR   r   r   .0fdar8   r)   r*   
<listcomp>  s     z1test_futures_to_delayed_array.<locals>.<listcomp>rM   )rN   rO   dask.array.utilsrT   arangerQ   concatenater   )r4   rT   npr_   Ar)   r   r*   test_futures_to_delayed_array   s   



 r   c                   s   t jjdd ttd }W d    n1 sw   Y  tdI d H  |jr-J ttdjdd}tdI d H  |jrDJ d S )Nr   re   r0   g{Gz?r>   )	rW   rv   rw   r   r!   r   asynciosleeprq   r4   r5   r6   r7   r8   r)   r)   r*   &test_local_get_with_distributed_active	  s   
r   c                 C  s*   t d t d ddlm} |  d S )Nr   rI   r   )test_to_hdf)rN   rO    dask.dataframe.io.tests.test_hdfr   )r4   r   r)   r)   r*   test_to_hdf_distributed  s   


r   rd   r0   r@   zHDF not multi-process safe)r%   strict)Zmarksr1   c                 C  s.   t d t d ddlm} |d |  d S )Nr   rI   r   )test_to_hdf_schedulers)rN   rO   r   r   )rd   r4   r   r)   r)   r*   !test_to_hdf_scheduler_distributed   s   

r   c                   s   t d}t d}|g dg dd}|j|dd}|jdd	d
jddd}| |jd I d H }	| |jd I d H }
|j	|
|	|
g|ddgddgdd d S )NrI   rJ   )r0   r>   rL   r@   )r0   r   r0   r   rm   r>   rc   r9   F)sortcount)Z	split_outr   r0   )rN   rO   rP   rh   groupbyZaggr   Z
partitionsrS   rT   rU   	set_index)r4   r5   r6   r7   r\   r]   r^   r`   rF   Zagg0Zagg1r)   r)   r*   test_serializable_groupby_agg9  s   

r   c                 C  sl   t dt d}}t t||}t t||}t t||}| |}t t|d}|jdddks4J d S )Nr0   r>   r1   dask.distributedre      )r   r   r   r   )r4   r8   r9   xxyyZxxyyZxxyy2Zxxyy3r)   r)   r*   test_futures_in_graphK  s   
r   c                 C  s   t d t d}t d t *}|jddd}|| ||}|j||| d |j|jks4J W d    d S 1 s?w   Y  d S )Nr   r   zarrrL   rL   r0   r0   chunksre   )rN   rO   r   zerosto_zarr	from_zarrrT   r   )r4   r   dr6   a2r)   r)   r*   test_zarr_distributed_roundtripW  s   




"r   c           	      C  s   t d t d}t d}t ?}d}|jd|d}|j|}|jj|j||j	|dd}|
| ||}|j||| d	 |j|jksIJ W d    d S 1 sTw   Y  d S )
Nr   r   r   r   r   r   r6   r   r   r   storemodere   )rN   rO   r   r   storageZDirectoryStorecreation
open_arrayr   r   r   r   rT   r   )	r4   r   r   r   r   r6   r5   rn   r   r)   r)   r*   3test_zarr_distributed_with_explicit_directory_stored  s&   




"r   c                 C  s   t d t d}t d}d}|jd|d}|j }|jj|j||j|dd}t j	t
d	d
 || W d    d S 1 sBw   Y  d S )Nr   r   r   r   r   r   r6   r   distributed schedulermatch)rN   rO   r   r   ZMemoryStorer   r   r   r   rZ   RuntimeErrorr   )r4   r   r   r   r6   r5   rn   r)   r)   r*   0test_zarr_distributed_with_explicit_memory_storez  s    



"r   c                 C  s|   t d t d}t d}d}|jd|d}|j||d}t jtdd || W d    d S 1 s7w   Y  d S )	Nr   r   r   r   r   r   r   r   )rN   rO   onesZ
zeros_likerZ   r   r   )r4   r   r   r   r6   rn   r)   r)   r*   #test_zarr_in_memory_distributed_err  s   


"r   c                   s<   t dd    j| ddksJ |  fddsJ d S )Nc                   S  s   dS r   r)   r)   r)   r)   r*   r         z.test_scheduler_equals_client.<locals>.<lambda>re   r0   c                   s   |   jS r   )Zstoryr2   r   r8   r)   r*   r     s    )r   r   r   r4   r)   r   r*   test_scheduler_equals_client  s   r   c                   s\   t td}| I d H }|j|jv sJ |js|jsJ tdd t|D s,J d S )Nr0   c                 s  s    | ]}|  V  qd S r   )doner   r)   r)   r*   	<genexpr>  s    ztest_await.<locals>.<genexpr>)	rW   r   r!   r   r2   rq   r3   allr   r   r)   r)   r*   
test_await  s   r   c                  C  s   dd } t |   d S )Nc                    s<   t td} | d }| I d H }t|j dksJ d S r   )rW   r   r!   r   len)r8   r9   rn   r)   r)   r*   r     s
   ztest_local_scheduler.<locals>.f)r   run)r   r)   r)   r*   test_local_scheduler  s   r   c                   sH  t d t d}t d}ddlm} ttdtdddg  fdd	}d
d }|jddd}	tjdd |	j	||j
d|jdd}
W d    n1 sQw   Y  tjdd |
j	||j
d|jdd}W d    n1 stw   Y  tjjdd | |I d H }W d    n1 sw   Y  |||dd  d S )Nr   r   r   r   r;   twor>   c                   s
     |  S r   r)   r   scaler)   r*   flaky_double     
z7test_annotations_blockwise_unpack.<locals>.flaky_doublec                 S  s   d|  S )Nr>   r)   r   r)   r)   r*   reliable_double  s   z:test_annotations_blockwise_unpack.<locals>.reliable_doubler1   )r=   r   )retriesr)   r   metaF)Zoptimization__fuse__activeg      @)rN   rO   r   rT   r   ZeroDivisionErrorr   rW   ZannotateZ
map_blocksarrayZfloat64rv   rw   r   )r4   r5   r6   r7   r   r   rT   r   r   r8   r9   rn   r)   r   r*   !test_annotations_blockwise_unpack  s&   


r   io)r   r   full)TFNc           
      C  s  t d}t d}d}d}|dkr|j||d}||}n#|dkr0|j||d}||}n|dkrB|j|d	|d}||d	}|d
7 }|d
7 }tjd|i* |  tj	
|j| }	t|	t|dukslJ |j||| d W d    d S 1 sw   Y  d S )Nr   r   )r=   r>   )r1   r@   r   r   r   r   r1   r>   ro   Fre   )rN   rO   r   r   r   rW   rv   rw   r   r   optimize__dask_keys__rE   dictrT   )
r4   r   rk   r   r   r   r   darrZnarrdskr)   r)   r*   test_blockwise_array_creation  s*   


"r   parquet-pyarrowparquet-fastparquetcsvhdfr=   )Zrerunsfrom_futuresc                 C  sH  t d}t d}| rt d |g dd tdd}|rF|jd d |jdd |jdd g}| |}	|j|	|d	 d
}
n|j	|dd}
|dkret d |

t| |t|}nl|dkrt d t t |
j
t|dd |jt|dd}W d    n1 sw   Y  n;|dkr|
jt|dd |tjt|d}n!|dkrt d t|d}|
|d ||d}ntd|dg d }|dg d }| s"tjd|i+ |  tj|j| }t|tt |ksJ |j!||dd W d    d S 1 sw   Y  d S d S )NrI   rJ   zdoesn't work yetrK   r=      rm   r1   r   r   rL   rc   r   Zpyarrowr   Zfastparquet)Zenginer   F)index*r   tablesZh5z/data*Zunreachabler8   ro   ru   )"rN   rO   rV   xfailrP   rg   ZilocrQ   rR   rh   Z
to_parquetstrZread_parquetwarnsFutureWarningZto_csvZread_csvospathjointo_hdfZread_hdfAssertionErrorrW   rv   rw   r   Z	dataframer   r   rE   r   boolrT   )r4   r   r   rk   r   r\   r]   r^   partsZfutsZddf0r`   fnr   r)   r)   r*   test_blockwise_dataframe_io  sP   


(




$r  c                 C  sv   t d}t d}|dg dd i}|j|ddd }|dk }t|}|d	ks,J |j|   d	ks9J d S )
NrI   rJ   r8   rK   r=   r>   rc   rL   r  )rN   rO   rP   rh   r   r8   r   sum)r4   r\   r]   r^   seriesrF   Z
series_lenr)   r)   r*   #test_blockwise_fusion_after_compute5  s   

r  c              	     sz   t d t d}t d  fdd}|j|d|dd dd jd	}| j| d
dI dH }|dks;J dS )zDTest pack/unpack of blockwise that includes a NumPy literal argumentr   r   c                      t | ju s	J | |S r   typeuint16astyper8   dtr   r)   r*   r  O     
z%test_blockwise_numpy_args.<locals>.fnr8     *   Nr   FZoptimize_graphrN   rO   	blockwiser   r  r   r  r4   r5   r6   r7   r   r  arrrG   r)   r  r*   test_blockwise_numpy_argsH  s   


r%  c              	     sz   t d t d}t d d fdd	}|j|d|dd j dd	}| j| d
dI dH }|dks;J dS )zLTest pack/unpack of blockwise that includes a NumPy literal keyword argumentr   r   Nc                   r  r   r  r  r  r)   r*   r  a  r  z'test_blockwise_numpy_kwargs.<locals>.fnr8   r  r  )r   r  Fr   r   r!  r#  r)   r  r*   test_blockwise_numpy_kwargsZ  s   


$r&  c                 C  s   t d}t d}||d}||g d}| }|| }t|\}|| }|g d}tj	ddi |
 }	|
 }
W d    n1 sPw   Y  |j|	| |j|
| d S )Nr   r   rL   )y      $@       @y      @      y       @      ?)y                y      @      @y      0@       ro   F)rN   rO   
from_arrayr   r   ZconjrW   r   rv   rw   r   testingZassert_equal)r4   r   r   uvZcvr8   r9   expectedZx_valueZy_valuer)   r)   r*   %test_blockwise_different_optimizationj  s   


r,  c                   s   t d}t d}t d}t d}dd }| dd d	}	| d
d d}
|j|d|jdddd|jdddd|	ddt|
d}|j|d|	didd}|j
ddd}|d  }| |  }| j|ddI dH }|dkswJ dS )z9Check pack/unpack of a HLG that has every type of Layers!r   rI   r   rJ   c                 S  s   | | | | S r   r)   )r8   r9   rn   	extra_argr)   r)   r*   r     r,   z&test_combo_of_layer_types.<locals>.addc                 S     | S r   r)   r   r)   r)   r*   r     r   z+test_combo_of_layer_types.<locals>.<lambda>r>   c                 S  r.  r   r)   r   r)   r)   r*   r     r   rL   r8   )rL   r0   r   NF)r   r   r-  r6   rc   rq   )rt   r      )rN   rO   r   r"  r   r   intrh   rP   r   shuffleZto_dask_arrayr  r   )r4   r5   r6   r7   r   r\   r   r]   r   r9   rn   r8   r^   rG   r)   r)   r*   test_combo_of_layer_types  s6   



r3  c              
     s   t d}t d  fdd} |g d} |g dg} j|d|d|d|jdd	}| j|d
d  j||| d dS )z1Test a blockwise operation with concatenated axesr   r   c                   s     |g dg | S )Nr   r0   r>   r   rm   r   r)   r*   r     s   z%test_blockwise_concatenate.<locals>.fr4  iZijT)r   r   Fr   re   N)rN   rO   r'  r   r"  r   r   rT   )r4   r   r   r8   r9   rn   r)   r5  r*   test_blockwise_concatenate  s"   


r7  c                   s~   t d}t d}|j|dtdidd}| |dddI d H }|d	 d	d	d
ks2J |d ddd
ks=J d S )NrI   rJ   r6   r1   r>   rc   c                 S  s   |S r   r)   )r8   Zpartition_infor)   r)   r*   r     r   z4test_map_partitions_partition_info.<locals>.<lambda>r   )numberdivisionr0   r=   r   )rN   rO   rh   rP   rg   r   map_partitions)r4   r5   r6   r7   r\   r]   r`   rG   r)   r)   r*   "test_map_partitions_partition_info  s   


r;  c                   s   t d}t d}|j|ttd|jddddddd	d
}||jtd 	 }|j
j |d< | j|jj|dgddI dH }dS )z.Copied from distributed (tests/test_client.py)rI   rJ   2   z
2020-01-01z
2020-09-01UTC)r   endZperiodstz)uid
enter_timer0   rc      dayF)columnsr  N)rN   rO   rh   rP   r   rg   Z
date_ranger@  isinr   rA  r  day_namer   ZcategoricalZ
categorize)r4   r5   r6   r7   r\   r]   r`   r)   r)   r*   test_futures_in_subgraphs  s"   

"rG  zmax_branch, expected_layer_type    r>   ) r0   )r
   nthreadsc                   s   t d}t d}| rt d G  fdddt |   I d H  |dtdi}|j|dd	}	|	j	dd
|d}
t
|
jjt
|	jj }|D ]}d|v r^t|
jj| |s^J qM| |
I d H   jrlJ d S )NrI   rJ   z"Checking layers doesn't make sensec                      s$   e Zd ZdZdd Z fddZdS )z?test_shuffle_priority.<locals>.EnsureSplitsRunImmediatelyPluginFc                 S  
   || _ d S r   )worker)r   rL  r)   r)   r*   setup  r   zEtest_shuffle_priority.<locals>.EnsureSplitsRunImmediatelyPlugin.setupc                   sV   |dkr%t dd | jjjD s'tdd t| jjjD r)d _tdd S d S d S )N	executingc                 s      | ]}d |j v V  qdS splitNr2   r   tsr)   r)   r*   r     s    

z]test_shuffle_priority.<locals>.EnsureSplitsRunImmediatelyPlugin.transition.<locals>.<genexpr>c                 s  rO  rP  rR  rS  r)   r)   r*   r     s    TzSplit tasks are not prioritized)	r   rL  staterN  anyr   readyfailurer   )r   r2   r   finishr    EnsureSplitsRunImmediatelyPluginr)   r*   
transition  s   zJtest_shuffle_priority.<locals>.EnsureSplitsRunImmediatelyPlugin.transitionN)r   r   r   rX  rM  r\  r)   rZ  r)   r*   r[    s    r[  r6   r  r1   rc   rq   )rt   
max_branchr2  )rN   rO   rV   skipr   r   rP   rg   rh   r2  rw   rW   rX   rE   r   rX  )r4   r5   r6   r7   r]  Zexpected_layer_typer\   r]   r^   r`   r|   Zshuffle_layersZ
layer_namer)   rZ  r*   test_shuffle_priority  s"   
	

r_  c           
        s   t d t dt d}t d}t d}| r#t d  fdd}|jd	d
 }|jddd }	| |j||	|j	dI dH  dS )z7Check that map_partitions can handle a dask array inputr   rI   r   rJ   zdask.datasetsz-roundtripping through arrays doesn't work yetc                   s$   t | jsJ t | jsJ | S r   )rE   rP   Zndarrayr   r6   r   r\   r)   r*   r        z'test_map_partitions_da_input.<locals>.fZ1d)freqr/  r0   r   r   N)
rN   rO   rV   r^  
timeseriesr   r   r   r:  _meta)
r4   r5   r6   r7   r   r]   datasetsr   r^   r$  r)   ra  r*   test_map_partitions_da_input  s   





"rg  c               	     s   t dt d   rt d fdd fdd} tjdd	d
diddddd+}tj|dd |   W d   n1 sEw   Y  W d   dS W d   dS 1 s]w   Y  dS )z[
    Check that map_partitions can handle a delayed
    partition of a dataframe input
    rI   rJ   z/map partitions can't deal with delayed properlyc                   s$   t |  jsJ t | jsJ | S r   )rE   rP   r`  )r\   r)   r*   r   0  rb  z'test_map_partitions_df_input.<locals>.fc                    sp    j dtdidd} |  d  } j dtdidd}|jddd}|j||d	d
  d S )Nr6   r1   r0   rc   r   r7   rq   )rr   rt   F)r   Zenforce_metadata)rh   rP   rg   ri   r   r2  r:  r   )Zitem_dfr`   Z	merged_dfr]   r   r\   r)   r*   main5  s   
z*test_map_partitions_df_input.<locals>.mainr   z:0Z	dashboardFr0   )Zscheduler_portZdashboard_addressZscheduler_kwargsasynchronousZ	n_workersrJ  	processes)rj  N)rN   rO   rV   r^  r   ZLocalClusterr   )ri  r   r)   rh  r*   test_map_partitions_df_input&  s,   


"rl  c                   s   t d}t d}t d |tdtdd}d|j_|j|dd}t 0}d	| }	|jd
|	dddidd}
| 	|
I d H  |j
j||d
|	ddd W d    d S 1 s]w   Y  d S )NrI   rJ   Z
sqlalchemyr1   r?   r  r0   rc   z
sqlite:///testTZechoF)r  Zengine_kwargsr   )Zcheck_divisions)rN   rO   rP   rg   r  namerh   r    Zto_sqlr   rS   rT   Zread_sql_table)r4   r5   r6   r7   r\   r]   r^   r`   r   urirF   r)   r)   r*   test_to_sql_engine_kwargsR  s&   



"rp  c           
        s   t d}t d}| rt d G dd d d}|jdg| td|d d	}|j|d
d}|jdd  fddddtd}	| 	|	I d H j
dksPJ d S )NrI   rJ   z!we don't offer a public reductionc                   @  s   e Zd Zdd ZdS )z0test_non_recursive_df_reduce.<locals>.SomeObjectc                 S  rK  r   )val)r   rq  r)   r)   r*   __init__w  r   z9test_non_recursive_df_reduce.<locals>.SomeObject.__init__N)r   r   r   rr  r)   r)   r)   r*   
SomeObjectv  s    rs     r0   r>   )r3   r  "   rc   c                 S  r.  r   r)   r   r)   r)   r*   r   ~  r   z.test_non_recursive_df_reduce.<locals>.<lambda>c                   s    |     S r   )r  r   rs  r)   r*   r     s    Fzcommit-dataset)chunkZ	aggregateZsplit_everytokenr   )rN   rO   rV   r^  ZSeriesrg   rh   Z	reductionobjectr   rq  )
r4   r5   r6   r7   r\   r]   Nr  Zdask_seriesrF   r)   rv  r*   test_non_recursive_df_reducen  s"   



r{  c                 C  sp   t d t d ztjjdddd dti}|jddd	}|	  W d S  t
y7   t d
 Y d S w )NrI   rJ   
2000-01-01
2000-07-0112hr   r>  rc  	timestampT)sortedz(dd.set_index triggered a recursion error)rN   rO   rW   rf  rd  Zreset_indexr  r  r   r   RecursionErrorZfail)r4   r`   r)   r)   r*   !test_set_index_no_resursion_error  s   


r  c                  C  sz   d} t jt| d tdd W d    n1 sw   Y  t jt| d tdd W d    d S 1 s6w   Y  d S )Nz	no Clientr   r   re   r   )rN   rZ   r   r   )msgr)   r)   r*   -test_get_scheduler_without_distributed_raises  s   "r  c                 C  sj   t  | jksJ d}tjt|d}t dd t dd W d    n1 s&w   Y  t|dks3J d S )NzkRunning on a single-machine scheduler when a distributed client is active might lead to unexpected results.r   threadsre   r   r>   )r   getrN   r  UserWarningr   )r4   Zwarning_messageZuser_warnings_ar)   r)   r*   *test_get_scheduler_with_distributed_active  s   
r  c              	   C  s   t  | jksJ tjjddM tt t  | jksJ W d    n1 s(w   Y  tjjd d t  | jks=J W d    n1 sGw   Y  W d    d S W d    d S 1 s_w   Y  d S )Nr  re   )r   r  rW   rv   rw   rN   r  r  r   r)   r)   r*   7test_get_scheduler_with_distributed_active_reset_config  s   "r  zscheduler, expected_classes))N)SerializableLockr  AcquirerProxy)r  )r  r  r  )rk  )r  r  r  c                 C  s   t d t d t jddd}t jddd}t jdd	d}|d
}|j|dgd}|jddd}t|||f|D ]\}}	t|| d}
|
jj|	ksMJ q;d S )Nr   rI   r   Requires dask.arrayr$   zdask.bagzRequires dask.bagrJ   Requires dask.dataframed   r8   rD  r  r>   rc   re   )	rN   rO   r   from_dask_arrayrg   zipr   r   r   )rf   Zexpected_classesr   r   r]   r   r`   dbag
collectionr+  rG   r)   r)   r*   test_get_scheduler_lock  s   
	

r  multiprocessing_method)ZspawnforkZ
forkserverc           	      C  s   t d t d t jddd}t jddd}|d}|j|d	gd
}tjddd}tjd|i  |||fD ]}t	|dd}t
|tjjsKJ q:W d    d S 1 sWw   Y  d S )Nr   rI   r   r  r$   rJ   r  r  r8   r  r  r>   rc   z)distributed.worker.multiprocessing-methodr   re   )rN   rO   r   r  r   rg   rW   rv   rw   r   rE   r   lockLock)	r4   r  r   r]   r   r`   r  r  rG   r)   r)   r*   #test_get_scheduler_lock_distributed  s   
	

"r  
lock_paramc                 C  st   t d t d t d tdd}tjjdddd	}|jt|d
|d W d   dS 1 s3w   Y  dS )z_https://github.com/dask/dask/issues/9972 and
    https://github.com/dask/dask/issues/10315
    rI   rJ   r  Zhd5)	extensionr|  r}  r~  r  z/ds_*)r2   r  N)rN   rO   r    rW   rf  rd  r  r  )r4   r  r   r`   r)   r)   r*   test_write_single_hdf  s   


"r  rf   r   )rv   rJ  c              
     s  t t tj tjjksJ tjj	dd, tj tj
jks'J t| jdddI d H }ztj tj
jks>J W | I d H  n| I d H  w t| jdddI d H }ztj |jkseJ W | I d H  n| I d H  w tj tj
jksJ t| jdddI d H }z?tj tj
jksJ |  tj }||jksJ W d    n1 sw   Y  tj tj
jksJ W | I d H  n| I d H  w t| jdddI d H }t| jdddI d H }z:|  tj |jksJ W d    n	1 sw   Y  tj |jksJ W | I d H  | I d H  n| I d H  | I d H  w tj tj
jks@J W d    n	1 sKw   Y  tj tjjks\J t| jdddI d H }zXtj |jkssJ tjj	dd4 tj tj
jksJ |  tj |jksJ W d    n	1 sw   Y  W d    n	1 sw   Y  W | I d H  n| I d H  w W d    d S 1 sw   Y  d S )Nr  re   FT)Zset_as_defaultrj  )rN   r  r  rW   baser   localZget_syncrv   rw   Zthreadedr  r   addresscloseZ
as_current)r5   r
   scZclient2r)   r)   r*   5test_get_scheduler_default_client_config_interleaving  sd     

 
+
"$r  c                   s:   t jddd}|dd }tdd |jD rJ d S )Nr  r1   rc   c                 S  s   | d S )N   r)   r   r)   r)   r*   r   7  s    z*test_bag_groupby_default.<locals>.<lambda>c                 s  s    | ]	}d |d v V  qdS )Zpartdr   Nr)   )r   kr)   r)   r*   r   8  s    z+test_bag_groupby_default.<locals>.<genexpr>)r   rg   r   rV  rW   )r4   r5   r6   r7   b2r)   r)   r*   test_bag_groupby_default4  s   r  )s
__future__r   rN   rO   r   r   r	  sys	functoolsr   operatorr   r   r   r   Zdistributed.utils_testr	   r
   r4   r   r   r   r   r   r   r   rW   Zdask.bagZbagr   r   r   r   Z	dask.baser   r   Zdask.blockwiser   Zdask.delayedr   r(   r   r   Zdask.layersr   r   Z
dask.utilsr   r   r   r    Zdask.utils_testr!   markZskipifplatformZ
pytestmarkfilterwarningsZignore_sync_scheduler_warningr+   r/   r<   rH   ra   rj   Zparametrizer   r   r   r   r   r   Zxfail_with_pyarrow_stringsr   paramr  r   r   r   r   r   r   r   r   r   r   r   r   Zskip_with_pyarrow_stringsZflakyr  r  r%  r&  r,  r3  r7  r;  rG  r_  rg  rl  rp  r{  r  r  r  r  r  r  r  r  r  r  r  r)   r)   r)   r*   <module>   sF   
$
	

,
		



/


#

(
,
		


=