
    >ie                       d dl mZ d dlZ ej        d          Zd dlZd dlZd dlZd dlm	Z	 d dl
mZ d dlmZmZmZ d dlmZ d dlmZ d d	lmZmZmZmZmZmZmZ d dlZd dlmZ d d
lmZmZm Z  d dl!m"Z"m#Z# d dl$m%Z% d dl&m'Z' d dl(m)Z)m*Z* d dl+m,Z,m-Z- d dl.m/Z/m0Z0m1Z1m2Z2 d dl3m4Z4 d e/e          v r e	ed          Z e	ed          Zej5        6                    ej7        dk    d          Z8ej5        9                    d          Z:d Z;d Z< ed          d             Z=d Z>d Z?d Z@ej5        A                    d ddg          d!             ZBej5        A                    d"d#d#gg          ej5        A                    d$ddg          d%                         ZCej5        A                    d&g d'          ej5        A                    d(d)d*g          d+                         ZDd, ZEd- ZFe: ed          d.                         ZGej5        jH        d/             ZIe:ej5        A                    d0d1 ejJ        d2ej5        K                    d3d4          5           ejJ        d6ej5        K                    d3d4          5          g          ej5        jH        d7                                     ZL ed          d8             ZMd9 ZNd: ZOd; ZPd< ZQ ed          d=             ZRd> ZS ed          d?             ZTej5        A                    d@g dA          ej5        A                    d g dB          dC                         ZUe:ej5        A                    d@dD ejJ        dEej5        jV        5          dF ejJ        dGej5        W                    dHI          5          g          ej5        A                    d g dB          ej5        A                    dJddg          dK                                                 ZXdL ZY ed          dM             ZZ ed          dN             Z[dO Z\ ed          dP             Z]dQ Z^ ed          dR             Z_ ed          dS             Z`ej5        A                    dTdUe-fdVe,fg           eddWgdVz  X          dY                         Za ed          dZ             Zbd[ Zcej5        9                    d           ed          d\                         Zd ed          d]             Zed^ Zfd_ Zgd` Zhda Ziej5        A                    dbg dc          dd             Zjej5        A                    deg df          dg             Zkej5        jV        ej5        A                    dhdejl        m                                g          di                         Zn edjdkig l          dm             Zo ed          dn             ZpdS )o    )annotationsNdistributed)partial)add)ClientSchedulerPluginWorkerPlugin)cleanup)client)clustercluster_fixturegen_clusterlooploop_in_threadpopenvarying)computedelayedpersist)compute_as_if_collectionget_scheduler)	Blockwise)Delayed)
futures_ofwait)ShuffleLayerSimpleShuffleLayer)get_named_argsget_scheduler_locktmpdirtmpfile)incshould_check_stateF)r#   win32zTThe teardown of distributed.utils_test.cluster_fixture fails on windows CI currentlyreasonzrignore:Running on a single-machine scheduler when a distributed client is active might lead to unexpected results.c                     ddl m}  d S )Nr   r   )dask.distributedr   r(   s    ;lib/python3.11/site-packages/dask/tests/test_distributed.pytest_can_import_clientr+   >   s    ''''''''    c                     ddl m}  d S )Nr   dumps)dask.distributed.protocolr/   r.   s    r*   test_can_import_nested_thingsr1   B   s    ////////r,   Tc                  K    t          t                    d          }t          |          \  }t          |           d {V  |j        |j        v s|j        |j        v sJ  t          t                    d          }t          |d          \  }}t          |           d {V  |j        |j        v s|j        |j        v sJ d S d S )N   
   )r   r"   r   r   keydata)	csabxx2yy2ones	            r*   test_persistr@   F   s      QAAJJER
r((NNNNNNN6QVrv////RAammGB
r((NNNNNNN6QVrv//////r,   c                d   t          d          dz   }|dz   }|dz   } t          |dd|gd| dfddg          }t          |d         d         t                    sJ t          |d         d         d         t                    sJ t          |d         d         t                    sJ d	g d
ddddgf}t	          | |k    sJ t          ||g| ddgd          }|d         d         |u sJ |d         d         |u sJ |d                                         dk    sJ |dd          ddgfk    sJ d S )Nr3         r9   r:      r   r9   r:      )r3   rC      )   rC   F)traverserH   )r   r   
isinstancer   r   )r7   r9   r:   resultsolress         r*   test_persist_nestedrN   U   sg   

QA	AA	AA1Aq!9--1vq1#>>FfQing.....fQinQ'11111fQilG,,,,,##VQ
4CFs""""
1a&!Qe
4
4
4Cq6!9>>>>q6!9>>>>q6>>q    qrr7q1#hr,   c                H   t          j        d          }t          j        d          }|                    dg di          } | j        ||g          }|                    |          }|j                            |                                |                    ||gd                     t          |j
        j        |j                 t                    sJ t          j        t                    5  |                    ddg          }d d d            d S # 1 swxY w Y   d S )	Npandasdask.dataframer;   r3   rC      r   axisr3   rC   )pytestimportorskip	DataFramescatterfrom_delayedutils	assert_eqr   concatrJ   dasklayers_namer   raises	TypeError)r7   pddddffuturesddfs         r*   !test_futures_to_delayed_dataframerh   h   sC   		X	&	&B		-	.	.B	sIII&	'	'BaiR!!G
//'
"
"CHs{{}}biiRqi&A&ABBB choci0)<<<<<	y	!	! & &ooq!f%%& & & & & & & & & & & & & & & & & &s   2DDDc                T   t          j        d          }t          j        d          }|                    dt          d          i          }|                    |d          }|                    |                                          }|j                            |||            d S )NrP   rQ   r;      rC   npartitions	scheduler)	rV   rW   rX   rangefrom_pandasrZ   
to_delayedr[   r\   )r7   rc   rd   re   rg   s        r*   test_from_delayed_dataframerr   y   s     
	X	&	&B		-	.	.B	sE"II&	'	'B
...
+
+C
//#..**
+
+CHsB!,,,,,r,   fusec                H   t          j        d          }t          j        d          }d}|                    t          |          t          |          d          }|                    t          |          t          |          d          }|                    ||          dz   }|                    |d          dz   }|dz  }|dz  }t
          j                            d	|i          5  |                    |d
gdd          }	|		                                 |	
                                                    d
          }
d d d            n# 1 swxY w Y   |j                            |
|                    |d
gd                              d
          d           d S )NrP   rQ   #   r;   r=   )r;   zrk   r4   rB   optimization.fuse.activer;   lefttasks)onhowshuffle)r{   r|   Fcheck_index)rV   rW   rX   ro   rp   r^   configsetmergeheadr   sort_valuesr[   r\   )r7   rs   rc   rd   sizedf1df2ddf1ddf2ddfmdfms              r*   $test_fused_blockwise_dataframe_merger      s   		X	&	&B		-	.	.B
 D
,,U4[[uT{{;;
<
<C
,,U4[[uT{{;;
<
<C>>#4>0025D>>#1>--2D2IC2IC	4d;	<	< . .zz$C5fgzFF		llnn((--. . . . . . . . . . . . . . . HSYYsu&Y11==cBBPU      s   .AEEEr{   r9   	broadcastc                b   t          j        d          }t          j        d          }|                    ddgdz  t          d          d          }|                    ddgt          d          d          }|                    |d          }|                    |d          }|                    ||||d	
          }	|	                                }
|j                            |
	                    d          |                    |||          	                    d          d           d S )NrP   rQ   r3   rC   rE   )r9   b_left)r9   b_rightrk   rz   )r{   r   r}   r9   )r{   Fr~   )
rV   rW   rX   ro   rp   r   r   r[   r\   r   )r7   r{   r   rc   rd   pdflpdfrdfldfrr   r   s              r*   test_dataframe_broadcast_merger      s    
	X	&	&B		-	.	.B<<q!fqjE!HH==>>D<<q!fq::;;D
..1.
-
-C
..1.
-
-C88Cy'8JJD
,,..CH
t##//44      r,   computation)Nr   dask.computezscheduler, use_distributed)NT)syncFc                `  	 t          j        d          	t          j        d           G d dt                    } | j         |                       	fd} | j        |          }|                                du sJ  | j        d           }|r|dk    snd
S J |            d
S )zvShould a collection use its default scheduler or the distributed
    scheduler when being computed within a task?
    rP   rQ   c                      e Zd Zd Zd ZdS )<test_default_scheduler_on_worker.<locals>.UpdateGraphCounterc                   K   d|_         d S )Nr   _update_graph_count)selfrn   s     r*   startzBtest_default_scheduler_on_worker.<locals>.UpdateGraphCounter.start   s      ,-I)))r,   c                &    |xj         dz  c_         d S Nr3   r   )r   rn   argskwargss       r*   update_graphzItest_default_scheduler_on_worker.<locals>.UpdateGraphCounter.update_graph   s    ))Q.))))r,   N)__name__
__module____qualname__r   r    r,   r*   UpdateGraphCounterr      s2        	. 	. 	.	/ 	/ 	/ 	/ 	/r,   r   c                 z   d}                      t          |           t          |           d          }                    |d          }|                               nUdk    rt	          j        |           n8dk    r0t          |j        |j        t          |j                             nJ d	S )
Nr4   rv   rC   rk   rm   r   r   FT)rX   ro   rp   r   r^   r   	__class__list)r   re   rg   r   rd   rc   rn   s      r*   fooz-test_default_scheduler_on_worker.<locals>.foo   s    \\d%++>>??nnRQn//KK)K,,,,N**L	22222666$sxch9     Ltr,   Tc                    | j         S Nr   )dask_schedulers    r*   <lambda>z2test_default_scheduler_on_worker.<locals>.<lambda>   s	    ~A r,   rC   r3   N)rV   rW   r   register_pluginsubmitrK   run_on_scheduler)
r7   r   use_distributedrn   r   r   rM   num_update_graphsrd   rc   s
    ` `    @@r*    test_default_scheduler_on_workerr      s   * 
	X	&	&B		-	.	.B/ / / / /_ / / / A((**+++       " !(3--C::<<4**AA  &5N!!!!!!=N!!!!!r,   c                    g d} | j         ||g          }t          j        |          }t          |          ||z   k    sJ d S )NrR   )rY   dbrZ   r   )r7   Lrf   r:   s       r*   test_futures_to_delayed_bagr      sP    		AaiAG
  A77a!er,   c                f   t          j        d          ddlm} t          j        d          }|                    d           | j        g          }                    fd|D             d          } ||                                |                    gd                     d S )N
dask.arrayr   r\   numpyrB   c                T    g | ]$}                     |j        j                   %S ))shapedtype)rZ   r   r   ).0fdar;   s     r*   
<listcomp>z1test_futures_to_delayed_array.<locals>.<listcomp>  s/    KKKa!'	9	9KKKr,   rT   )rV   rW   dask.array.utilsr\   arangerY   concatenater   )r7   r\   nprf   Ar   r;   s        @@r*   test_futures_to_delayed_arrayr      s    		\	*	*B******		W	%	%B
		!AaiAG
KKKKK7KKKRS 	 	 	A Iaiikk2>>1a&q>99:::::r,   c                  K   t           j                            d          5   t          t                    d                                          }d d d            n# 1 swxY w Y   t          j        d           d {V  |j        rJ  t          t                    d                              d          }t          j        d           d {V  |j        rJ d S )Nr   rm   r3   g{Gz?rC   )	r^   r   r   r   r"   r   asynciosleeprz   r7   r8   r9   r:   r;   s        r*   &test_local_get_with_distributed_activer     s      
6	*	* & &GCLLOO##%%& & & & & & & & & & & & & & &
-

wQ&11A
-

ws   0AA#&A#c                v    t          j        d           t          j        d           ddlm}  |             d S )Nr   rP   r   )test_to_hdf)rV   rW    dask.dataframe.io.tests.test_hdfr   )r7   r   s     r*   test_to_hdf_distributedr     sE    
   
!!!<<<<<<KMMMMMr,   rl   r3   rE   zHDF not multi-process safe)r&   strict)marksr4   c                z    t          j        d           t          j        d           ddlm}  |d |            d S )Nr   rP   r   )test_to_hdf_schedulers)rV   rW   r   r   )rl   r7   r   s      r*   !test_to_hdf_scheduler_distributedr     sQ    "    
!!!GGGGGG4-----r,   c                V  K   t          j        d          }t          j        d          }|                    g dg dd          }|                    |d          }|                    dd	
                              dd          } | j        |j        d                    d {V }	 | j        |j        d                    d {V }
|j        	                    |
                    |	|
g          |                    ddgddgd                              d                     d S )NrP   rQ   )r3   rC   rS   rE   )r3   r   r3   r   rv   rC   rk   r=   F)sortcount)	split_outr   r3   )rV   rW   rX   rp   groupbyaggr   
partitionsr[   r\   r]   	set_index)r7   r8   r9   r:   rc   rd   re   rg   rK   agg0agg1s              r*   test_serializable_groupby_aggr   7  sA     		X	&	&B		-	.	.B	LLL|||<<	=	=B
...
+
+C[[5[))--g-CCF 6,Q/00000000D6,Q/00000000DH
		4,
Aq6A//00::3??    r,   c                   t          d          t          d          }} t          t                    ||          } t          t                    ||          } t          t                    ||          } | j        |          } t          t                    |d          }|                    d          dk    sJ d S )Nr3   rC   r4   dask.distributedrm      )r   r   r   r   )r7   r;   r=   xxyyxxyyxxyy2xxyy3s           r*   test_futures_in_graphr   I  s    1::wqzzqA	a		B	a		B73<<BDAIdOOEGCLL##E==#5=66:RRRRRRRr,   c                |   t          j        d          }t          j        d           t                      5 }|                    dd          }|                    |           |                    |          }|                    |||            |j        |j        k    sJ 	 d d d            d S # 1 swxY w Y   d S )Nr   zarrrS   rS   r3   r3   chunksrm   )rV   rW   r    zerosto_zarr	from_zarrr\   r   )r7   r   dr9   a2s        r*   test_zarr_distributed_roundtripr   U  s    		\	*	*B
	 %QHHVFH++			!\\!__
Qa(((yAH$$$$$% % % % % % % % % % % % % % % % % %s   A,B11B58B5c                H   t          j        d          }t          j        d          }d}|                    d|          }|                    ||          }t          j        t
          d          5  |                    |           d d d            d S # 1 swxY w Y   d S )Nr   r   r   r   r   zdistributed schedulermatch)rV   rW   ones
zeros_likera   RuntimeErrorr   )r7   r   r   r   r9   rw   s         r*   #test_zarr_in_memory_distributed_errr  a  s    		\	*	*Bv&&DF
v&&A&))A	|+B	C	C	C  			!                 s   4BBBc                     t          d                                           |           dk    sJ  | j        fd          sJ d S )Nc                     dS r   r   r   r,   r*   r   z.test_scheduler_equals_client.<locals>.<lambda>n       r,   rm   r3   c                8    |                      j                  S r   )storyr5   )r   r;   s    r*   r   z.test_scheduler_equals_client.<locals>.<lambda>p  s    ^5I5I!%5P5P r,   )r   r   r   )r7   r;   s    @r*   test_scheduler_equals_clientr  m  sb    		A99q9!!Q&&&&1PPPPQQQQQQQr,   c                  K    t          j        t                    d          }|                                 d {V }|j        |j        v sJ |j        s	|j        sJ t          d t          |          D                       sJ d S )Nr3   c              3  >   K   | ]}|                                 V  d S r   )done)r   r   s     r*   	<genexpr>ztest_await.<locals>.<genexpr>y  s*      //Aqvvxx//////r,   )	r^   r   r"   r   r5   rz   r6   allr   r   s        r*   
test_awaitr  s  s      S!AiikkA5AG6QV//A//////////r,   c                 D    d } t          j         |                        d S )Nc                    K    t          j        t                    d          } | dz   }|                                 d {V }t	          |j                   dk    sJ d S r   )r^   r   r"   r   len)r;   r=   rw   s      r*   r   ztest_local_scheduler.<locals>.f}  sf      DLa  E))++16{{ar,   )r   run)r   s    r*   test_local_schedulerr  |  s/          Kr,   c                X  K   t          j        d          }t          j        d          }ddlm} t	          t          d          t          d          ddg          fd}d	 }|                    d
d          }	t          j        d          5  |		                    ||
                    d|j                            }
d d d            n# 1 swxY w Y   t          j        d          5  |
	                    ||
                    d|j                            }d d d            n# 1 swxY w Y   t          j                            d          5   | j        |           d {V }d d d            n# 1 swxY w Y    |||                    d
          dz             d S )Nr   r   r   r   r?   tworC   c                                 | z  S r   r   )r;   scales    r*   flaky_doublez7test_annotations_blockwise_unpack.<locals>.flaky_double  s    uww{r,   c                    d| z  S )NrC   r   r;   s    r*   reliable_doublez:test_annotations_blockwise_unpack.<locals>.reliable_double  s    1ur,   r4   )rB   r   )retriesr   r   metaF)optimization__fuse__activeg      @)rV   rW   r   r\   r   ZeroDivisionErrorr   r^   annotate
map_blocksarrayfloat64r   r   r   )r7   r8   r9   r:   r   r   r\   r  r  r;   r=   rw   r  s               @r*   !test_annotations_blockwise_unpackr&    s     		\	*	*B		W	%	%B****** &u--/@/G/GANOOE       	4  A 
q	!	!	! L LLLBHHRrzH,J,JLKKL L L L L L L L L L L L L L L	q	!	!	! O OLLrxx"*x/M/MLNNO O O O O O O O O O O O O O O 
E	:	:  !)A,,               Ias"#####s6   2CCC12D//D36D3E==FFio)r   r   full)TFNc                   t          j        d          }t          j        d          }d}d}|dk    r-|                    ||          }|                    |          }ng|dk    r-|                    ||          }|                    |          }n4|dk    r.|                    |d	|          }|                    |d	          }|d
z  }|d
z  }t
          j                            d|i          5  |                                 t
          j	        
                    |j        |                                          }	t          |	t                    |duk    sJ |                    |||            d d d            d S # 1 swxY w Y   d S )Nr   r   )rB   rC   )r4   rE   r   r   r   r(  r4   rC   rx   Frm   )rV   rW   r   r   r(  r^   r   r   r   r$  optimize__dask_keys__rJ   dictr\   )
r7   r'  rs   r   r   r   r   darrnarrdsks
             r*   test_blockwise_array_creationr0    s    
	W	%	%B		\	*	*BFE	V||wwuVw,,wwu~~	wxxfx--xx	vwwubw00wwub!!AIDAID	4d;	<	< . .j!!$)T-?-?-A-ABB#t$$U):;;;;
T41---. . . . . . . . . . . . . . . . . .s   2BF  FFparquet-pyarrowzparquet-fastparquetcsvhdfrB   )rerunsfrom_futuresc                   t          j        d          }t          j        d          }|                    g ddz  t          d          d          }|rZ|j        d d         |j        dd         |j        dd         g} | j        |          }	|                    |	|d         	          }
n|                    |d
          }
|                    d          r||dk    rt          j        d           d}nt          j        d           d}|
	                    t          |          |           |                    t          |          |          }n|dk    re|
                    t          |          d           |                    t          j                            t          |          d                    }nc|dk    r]t          j        d           t           |j        d                    }|
                    |d           |                    |d          }|dg         dz   }|dg         dz   }t&          j                            d|i          5  |                                 t&          j                            |j        |                                          }t5          |t6                    t9          |          k    sJ |                    ||d           d d d            d S # 1 swxY w Y   d S )NrP   rQ   rR   rB      rv   r4   r   r  rS   rk   parquetr1  zpyarrow.parquetpyarrowfastparquet)enginer2  F)index*r3  tablesh5z/data*r;   rx   r~   )rV   rW   rX   ro   ilocrY   rZ   rp   
startswith
to_parquetstrread_parquetto_csvread_csvospathjointo_hdfread_hdfr^   r   r   r   	dataframer*  r+  rJ   r,  boolr\   )r7   r    r'  rs   r5  rc   rd   re   partsfutsddf0r;  rg   fnr/  s                  r*   test_blockwise_dataframe_iorR    s     
	X	&	&B		-	.	.B	IIIMb		::	;	;B 1!bgadmRWRU^<qyt%(33~~ba~00	}}Y (""" 1222FF..."FFF333ooc&kk&o99	uCKKu---kk"',,s6{{C8899	uH%%%T""##B!!!kk"h''	SERB
se*r/C	4d;	<	< 1 1n%%ch0A0A0C0CDD#t$$T

2222
S"%0001 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1s   BK77K;>K;c                p   t          j        d          }t          j        d          }|                    dg ddz  i          }|                    |d          d         }|dk     }t	          |          }|d	k    sJ |j        |                                                                         d	k    sJ d S )
NrP   rQ   r;   rR   rB   rC   rk   rS   r7  )rV   rW   rX   rp   r  r;   r   sum)r7   rc   rd   re   seriesrK   
series_lens          r*   #test_blockwise_fusion_after_computerW    s     
	X	&	&B		-	.	.B 
sIIIM*	+	+B^^BA^..s3FaZF
 VJ4  !%%''2------r,   c           	     X  K   t          j        d          }t          j        d          fd}|                    |d|                    d          d                    d          dj                  } | j        |                                d	
           d{V }|dk    sJ dS )zDTest pack/unpack of blockwise that includes a NumPy literal argumentr   r   c                ^    t          |          j        u sJ |                     |          S r   typeuint16astyper;   dtr   s     r*   rQ  z%test_blockwise_numpy_args.<locals>.fn  ,    Bxx29$$$$xx||r,   r;     *   Nr  Foptimize_graphrV   rW   	blockwiser   r\  r   rT  	r7   r8   r9   r:   r   rQ  arrrM   r   s	           @r*   test_blockwise_numpy_argsri    s       
	\	*	*B		W	%	%B     ,,
CRYYr]]D	   C 	#''))E:::
:
:
:
:
:
:C$;;;;;;r,   c           
     X  K   t          j        d          }t          j        d          dfd	}|                    |d|                    d          dj                            d                    } | j        |                                d	
           d{V }|dk    sJ dS )zLTest pack/unpack of blockwise that includes a NumPy literal keyword argumentr   r   Nc                ^    t          |          j        u sJ |                     |          S r   rZ  r^  s     r*   rQ  z'test_blockwise_numpy_kwargs.<locals>.fn,  r`  r,   r;   ra  rb  )r   r_  Frc  r   re  rg  s	           @r*   test_blockwise_numpy_kwargsrl  &  s       
	\	*	*B		W	%	%B      ,,r3s")		RT,
V
VC	#''))E:::
:
:
:
:
:
:C$;;;;;;r,   c                   t          j        d          }t          j        d          }|                    |                    d                    }|                    |                    g d                    }|                                }||z  }t          j        |          \  }||z  }|                    g d          }t          j        	                    ddi          5  |
                                }	|
                                }
d d d            n# 1 swxY w Y   |j                            |	|           |j                            |
|           d S )Nr   r   rS   )y      $@       @y      @      y       @      ?)y                y      @      @y      0@       rx   F)rV   rW   
from_arrayr   r$  conjr^   r*  r   r   r   testingassert_equal)r7   r   r   uvcvr;   r=   expectedx_valuey_values              r*   %test_blockwise_different_optimizationrx  5  sl   
 
	\	*	*B		W	%	%B
biill##A
bhh88899::A	
B	BAM"ER	BAxx11122H	4e<	=	=  ))++))++               JGX...JGX.....s   ')DD #D c                  K   t          j        d          }t          j        d          }t          j        d          }t          j        d          }d } | j        d d          }	 | j        d d	          }
|                    |d
|                    dd          d
|                    dd          d
|	ddt          |
          }|                    |                    d|	                    d	          i          d	          }|
                    dd          }|d                                         }|                                |                                z   } | j        |d           d{V }|dk    sJ dS )z9Check pack/unpack of a HLG that has every type of Layers!r   rQ   r   rP   c                    | |z   |z   |z   S r   r   )r;   r=   rw   	extra_args       r*   r   z&test_combo_of_layer_types.<locals>.addT  s    1uqy9$$r,   c                    | S r   r   r  s    r*   r   z+test_combo_of_layer_types.<locals>.<lambda>W      1 r,   rC   c                    | S r   r   r  s    r*   r   z+test_combo_of_layer_types.<locals>.<lambda>X  r}  r,   rS   r;   )rS   r3   r   NF)r   r   r{  r9   rk   rz   )r}   rc     )rV   rW   r   rf  r   r   intrp   rX   r   r}   to_dask_arrayrT  r   )r7   r8   r9   r:   r   rd   r   rc   r   r=   rw   r;   re   rM   s                 r*   test_combo_of_layer_typesr  K  s      
	\	*	*B		-	.	.B		W	%	%B		X	&	&B% % % 	a  Aa  A

d##
T""	 	 	 	A 
c299Q<<%899q	I	IB	C	)	)B	C			 	 B
%%''BFFHH
C	#e444
4
4
4
4
4
4C"999999r,   c           
        t          j        d          t          j        d          }fd}                    |                    g d                    }                    |                    g dg                    }                    |d|d|d|j        d          } | j        |d	
                               |||            dS )z1Test a blockwise operation with concatenated axesr   r   c                :                         |g dg           | S )Nr   r3   rC   r   )r;   r=   r   s     r*   r   z%test_blockwise_concatenate.<locals>.fu  s"    
Q$$$r,   r  iijT)r   r   Frc  rm   N)rV   rW   rn  r$  rf  r   r   r\   )r7   r   r   r;   r=   rw   r   s         @r*   test_blockwise_concatenater  p  s    		\	*	*B		W	%	%B     	bhhyyy))**A
bhh			{++,,A
						g 	 		 		A AIa&&&&LLAL#####r,   c                f  K   t          j        d          }t          j        d          }|                    |                    dt	          d          i          d          } | j        |                    dd                     d {V }|d         ddd	k    sJ |d
         d
dd	k    sJ d S )NrQ   rP   r9   r4   rC   rk   c                    |S r   r   )r;   partition_infos     r*   r   z4test_map_partitions_partition_info.<locals>.<lambda>  s    . r,   r   )numberdivisionr3   rB   r   )rV   rW   rp   rX   ro   r   map_partitions)r7   r8   r9   r:   rd   rc   rg   rM   s           r*   "test_map_partitions_partition_infor    s      		-	.	.B		X	&	&B
..sE"II&677Q.
G
GC	HHHII       C q6q111111q6q11111111r,   c                  K   t          j        d          }t          j        d          }|                    |                    t	          t          d          |                    dddd                              d	
          }||j                            t          d                             	                                }|j
        j                                        |d<    | j        |j        j        |dgd           d{V }dS )z.Copied from distributed (tests/test_client.py)rQ   rP   2   z
2020-01-01z
2020-09-01UTC)r   endperiodstz)uid
enter_timer3   rk      dayF)columnsr<  N)rV   rW   rp   rX   r,  ro   
date_ranger  isinr   r  r_  day_namer   categorical
categorize)r7   r8   r9   r:   rd   rc   rg   s          r*   test_futures_in_subgraphsr    s      
	-	.	.B		X	&	&B
..
"II==&L" )    	
 	
   
 
C cgll599%%
&
.
.
0
0C"++--CJ2C%PUVVV
V
V
V
V
V
VCCCr,   zmax_branch, expected_layer_type    rC   ) r3   )r   nthreadsc                Z  K   t          j        d          }t          j        d          } G fddt                     | j                                d {V  |                    dt          d          i          }|                    |d          }	|	                    dd	|
          }
t          |
j	        j
                  t          |	j	        j
                  z
  }|D ]$}t          |
j	        j
        |         |          sJ % | j        |
           d {V  j        rJ d S )NrP   rQ   c                  $    e Zd ZdZd Z fdZdS )?test_shuffle_priority.<locals>.EnsureSplitsRunImmediatelyPluginFc                    || _         d S r   )worker)r   r  s     r*   setupzEtest_shuffle_priority.<locals>.EnsureSplitsRunImmediatelyPlugin.setup  s     DKKKr,   c                   |dk    rst          d | j        j        j        D                       sMt	          d t          | j        j        j                  D                       rd_        t          d          d S d S d S )N	executingc              3  (   K   | ]}d |j         v V  dS splitNr5   r   tss     r*   r  z]test_shuffle_priority.<locals>.EnsureSplitsRunImmediatelyPlugin.transition.<locals>.<genexpr>  s:       1 1&(26!1 1 1 1 1 1r,   c              3  (   K   | ]}d |j         v V  dS r  r  r  s     r*   r  z]test_shuffle_priority.<locals>.EnsureSplitsRunImmediatelyPlugin.transition.<locals>.<genexpr>  s)      QQRw"&(QQQQQQr,   TzSplit tasks are not prioritized)	r  r  stater  anyr   readyfailurer  )r   r5   r   finishr    EnsureSplitsRunImmediatelyPlugins        r*   
transitionzJtest_shuffle_priority.<locals>.EnsureSplitsRunImmediatelyPlugin.transition  s    $$S 1 1,0K,=,G1 1 1 . .$ QQ48I8O3P3PQQQQQ J?C4<&'HIII %$$$J Jr,   N)r   r   r   r  r  r  )r  s   r*   r  r    sI        	! 	! 	!	J 	J 	J 	J 	J 	J 	Jr,   r  r9   ra  r4   rk   rz   )r}   
max_branch)rV   rW   r	   r   rX   ro   rp   r}   r   r^   r_   rJ   r   r  )r7   r8   r9   r:   r  expected_layer_typerc   rd   re   rg   r   shuffle_layers
layer_namer  s                @r*   test_shuffle_priorityr    sr      
	X	&	&B		-	.	.BJ J J J J J J< J J J !
<<>>
?
???????? 
sE$KK(	)	)B
...
,
,C;;sG
;CCD)**S-A-AAN$ M M
$)*:68KLLLLLL
!)D///777777r,   c                  	
K   t          j        d          	t          j        d          
t          j        d          }t          j        d          }	
fd}|                    d                                          }|                    dd	
                                          } | j        |                    |||j                             d{V  dS )z7Check that map_partitions can handle a dask array inputr   rP   r   zdask.datasetsc                d    t          | j                  sJ t          |j                  sJ | S r   )rJ   rX   ndarray)r   r9   r   rc   s     r*   r   z'test_map_partitions_da_input.<locals>.f  s8    !R\*****!RZ(((((r,   1d)freqr  r3   r   r  N)rV   rW   
timeseriesr   r   r   r  _meta)r7   r8   r9   r:   r   datasetsr   re   rh  r   rc   s            @@r*   test_map_partitions_da_inputr    s       
	W	%	%B		X	&	&B		\	*	*B"?33H     
 
		$		'	'	/	/	1	1B
''$q'
!
!
)
)
+
+C
!)B%%a28%<<
=
==========r,   c            	     h   t          j        d          t          j        d          fdfd} t                              ddddidd	d	d
          5 }t                              |d          5   |              ddd           n# 1 swxY w Y   ddd           dS # 1 swxY w Y   dS )z[
    Check that map_partitions can handle a delayed
    partition of a dataframe input
    rP   rQ   c                d    t          | j                  sJ t          |j                  sJ | S r   )rJ   rX   )r   r9   rc   s     r*   r   z'test_map_partitions_df_input.<locals>.f  s8    !R\*****!R\*****r,   c                                                             dt          d          i          d          } |                                 d                                         }                                         dt          d          i          d          }|                    dd          }|                    ||d	
                                           d S )Nr9   r4   r3   rk   r   r:   rz   )r{   r}   F)r  enforce_metadata)rp   rX   ro   rq   r   r}   r  r   )item_dfrg   	merged_dfrd   r   rc   s      r*   mainz*test_map_partitions_df_input.<locals>.main  s    ..sE"II.>!?!?Q.OO  ""1%--//NN2<<eBii0@#A#AqNQQ	 %%g%>>	  sU 	! 	
 	

')))))r,   r   z:0	dashboardFr3   )scheduler_portdashboard_addressscheduler_kwargsasynchronous	n_workersr  	processes)r  N)rV   rW   r   LocalClusterr   )r  r   rd   r   rc   s     @@@r*   test_map_partitions_df_inputr    sz   
 
	X	&	&B		-	.	.B    

 
 
 
 
 
 
 
	!	!
 %u- 
" 
 
  
e<< 	 	DFFF	 	 	 	 	 	 	 	 	 	 	 	 	 	 	                 s6   B'8BB'B	B'B	B''B+.B+c           	     L  K   t          j        d          }t          j        d          }t          j        d           |                    t          d          t          d          d          }d|j        _        |                    |d          }t                      5 }d	| }	|                    d
|	dddid          }
 | j	        |
           d {V  |j
                            ||                    d
|	d          d           d d d            d S # 1 swxY w Y   d S )NrP   rQ   
sqlalchemyr4   rD   r<  r3   rk   z
sqlite:///testTechoF)r<  engine_kwargsr   )check_divisions)rV   rW   rX   ro   r<  namerp   r!   to_sqlr   r[   r\   read_sql_table)r7   r8   r9   r:   rc   rd   re   rg   r   urirK   s              r*   test_to_sql_engine_kwargsr    s|      
	X	&	&B		-	.	.B
%%%	E"IIE"II66	7	7BBHM
...
+
+C	 
a1CtFE?E  
 
 ai
fc733! 	 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
s   "A*DD Dc                  
K   t          j        d          }t          j        d          } G d d          
d}|                    dg|z  t          d|dz                       }|                    |d	
          }|                    d 
fdddt                    }	 | j        |	           d {V j        dk    sJ d S )NrQ   rP   c                      e Zd Zd ZdS )0test_non_recursive_df_reduce.<locals>.SomeObjectc                    || _         d S r   )val)r   r  s     r*   __init__z9test_non_recursive_df_reduce.<locals>.SomeObject.__init__;  s    DHHHr,   N)r   r   r   r  r   r,   r*   
SomeObjectr  :  s#        	 	 	 	 	r,   r     r3   rC   )r6   r<  "   rk   c                    | S r   r   r  s    r*   r   z.test_non_recursive_df_reduce.<locals>.<lambda>B  r  r,   c                b     |                                                                             S r   )rT  )r;   r  s    r*   r   z.test_non_recursive_df_reduce.<locals>.<lambda>C  s!    JJquuww{{}}55 r,   Fzcommit-dataset)chunk	aggregatesplit_everytokenr  )	rV   rW   Seriesro   rp   	reductionobjectr   r  )r7   r8   r9   r:   rd   rc   NrU  dask_seriesrK   r  s             @r*   test_non_recursive_df_reducer  3  s      
	-	.	.B		X	&	&B        	AYYQC!G5AE??Y;;F..R.88K""k5555 #  F !)F########(C//////r,   c                l   t          j        d           	 t          j                            ddd                                                              dt          i          }|                    dd          }|	                                 d S # t          $ r t          j        d	           Y d S w xY w)
NrQ   
2000-01-01
2000-07-0112hr   r  r  	timestampT)sortedz(dd.set_index triggered a recursion error)rV   rW   r^   r  r  reset_indexr]  rC  r   r   RecursionErrorfail)r7   rg   s     r*   !test_set_index_no_resursion_errorr  L  s    
()))	@M$$<\PU$VV[]]V[#&'' 	
 mmKm55 @ @ @>??????@s   A9B B32B3c                    d} t          j        t          |           5  t          d           d d d            n# 1 swxY w Y   t          j        t          |           5  t          d           d d d            d S # 1 swxY w Y   d S )Nz	no Clientr   r   rm   r   )rV   ra   r  r   )msgs    r*   -test_get_scheduler_without_distributed_raisesr  [  s   
C	|3	/	/	/ 4 4 233334 4 4 4 4 4 4 4 4 4 4 4 4 4 4 
|3	/	/	/ / /..../ / / / / / / / / / / / / / / / / /s   ;??!A??BBc                   t                      | j        k    sJ d}t          j        t          |          5 }t          d           t          d           d d d            n# 1 swxY w Y   t          |          dk    sJ d S )NzkRunning on a single-machine scheduler when a distributed client is active might lead to unexpected results.r   threadsrm   r   rC   )r   getrV   warnsUserWarningr  )r7   warning_messageuser_warnings_as      r*   *test_get_scheduler_with_distributed_activer  d  s    ??ae####	6  
k	9	9	9 (_	****''''( ( ( ( ( ( ( ( ( ( ( ( ( ( ( 1$$$$$$s   !A$$A(+A(c                   t                      | j        k    sJ t          j                            d          5  t          j        t                    5  t                      | j        k    sJ 	 d d d            n# 1 swxY w Y   t          j                            d           5  t                      | j        k    sJ 	 d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nr  rm   )r   r	  r^   r   r   rV   r
  r  )r7   s    r*   7test_get_scheduler_with_distributed_active_reset_configr  p  s   ??ae####	9	-	- , ,\+&& 	, 	, ??ae+++++	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	,[__t_,, 	, 	, ??ae+++++	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	,, , , , , , , , , , , , , , , , , ,sY   C%A;/C%;A?	?C%A?	#C%&CC%C	C%C	C%%C),C)zscheduler, expected_classes))N)SerializableLockr  AcquirerProxy)r  )r  r  r  )r  )r  r  r  c                   t          j        dd          }t          j        dd          }t          j        dd          }|                    d          }|                    |d	g
          } |j        dd          }t          |||f|          D ](\  }}	t          ||           }
|
j        j        |	k    sJ )d S )Nr   Requires dask.arrayr%   zdask.bagzRequires dask.bagrQ   Requires dask.dataframed   r;   r  r  rC   rk   rm   )	rV   rW   r   from_dask_arrayro   zipr   r   r   )rn   expected_classesr   r   rd   r-  rg   dbag
collectionru  rM   s              r*   test_get_scheduler_lockr  y  s     
	\2G	H	H	HB		Z0C	D	D	DB		-6O	P	P	PB776??D


TC5

1
1C28CQ'''D #S$$57G H H 2 2
H yAAA}%111112 2r,   multiprocessing_method)spawnfork
forkserverc                   t          j        dd          }t          j        dd          }|                    d          }|                    |dg          }t	          j        d	d
          }t          j                            d|i          5  |||fD ]4}t          |d          }t          |t          j        j                  sJ 5	 d d d            d S # 1 swxY w Y   d S )Nr   r  r%   rQ   r  r  r;   r  r  rC   rk   z)distributed.worker.multiprocessing-methodr   rm   )rV   rW   r   r  r   ro   r^   r   r   r   rJ   r   lockLock)	r7   r  r   rd   r-  rg   r  r  rM   s	            r*   #test_get_scheduler_lock_distributedr&    s?    
	\2G	H	H	HB		-6O	P	P	PB776??D


TC5

1
1C8CQ'''D		46LM
 
 : : d+ 	: 	:J$Z=IIICc;#3#8999999	:: : : : : : : : : : : : : : : : : :s   ;CC!C
lock_paramc                6   t          j        d           t          j        d           t          d          5 }t          j                            ddd          }|                    t          |          d	|
           ddd           dS # 1 swxY w Y   dS )z_https://github.com/dask/dask/issues/9972 and
    https://github.com/dask/dask/issues/10315
    rQ   r>  hd5)	extensionr  r  r  r  z/ds_*)r5   r$  N)rV   rW   r!   r^   r  r  rJ  rC  )r7   r'  r   rg   s       r*   test_write_single_hdfr+    s     ()))
!!!	5	!	!	! 9Qm&&\|RW&XX

3q66wZ
8889 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9s   ABBBrn   r   )r   r  c                   K   t          j        t                    5  t          j                                        t          j        j        k    sJ t          j        	                    d          5  t          j                                        t          j
        j        k    sJ t          | j        dd           d {V }	 t          j                                        t          j
        j        k    sJ 	 |                                 d {V  n# |                                 d {V  w xY wt          | j        dd           d {V }	 t          j                                        |j        k    sJ 	 |                                 d {V  n# |                                 d {V  w xY wt          j                                        t          j
        j        k    sJ t          | j        dd           d {V }	 t          j                                        t          j
        j        k    sJ |                                5  t          j                                        }||j        k    sJ 	 d d d            n# 1 swxY w Y   t          j                                        t          j
        j        k    sJ 	 |                                 d {V  n# |                                 d {V  w xY wt          | j        dd           d {V }t          | j        dd           d {V }	 |                                5  t          j                                        |j        k    sJ 	 d d d            n# 1 swxY w Y   t          j                                        |j        k    sJ 	 |                                 d {V  |                                 d {V  n9# |                                 d {V  |                                 d {V  w xY wt          j                                        t          j
        j        k    sJ 	 d d d            n# 1 swxY w Y   t          j                                        t          j        j        k    sJ t          | j        dd           d {V }	 t          j                                        |j        k    sJ t          j        	                    d          5  t          j                                        t          j
        j        k    sJ |                                5  t          j                                        |j        k    sJ 	 d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   |                                 d {V  n# |                                 d {V  w xY w	 d d d            d S # 1 swxY w Y   d S )Nr  rm   FT)set_as_defaultr  )rV   r
  r  r^   baser   localget_syncr   r   threadedr	  r   addressclose
as_current)r8   r   scclient2s       r*   5test_get_scheduler_default_client_config_interleavingr7    s      
k	"	" 7! 7!y&&((DJ,?????[__y_11 )	B )	B9**,,0AAAAA!!)EPTUUUUUUUUUF%y..00DM4EEEEEEllnn$$$$$$$$fllnn$$$$$$$$!!)DtTTTTTTTTTF%y..00FJ>>>>>llnn$$$$$$$$fllnn$$$$$$$$9**,,0AAAAA "!)EPTUUUUUUUUUF%y..00DM4EEEEE&&(( , ,0022B+++++, , , , , , , , , , , , , , , y..00DM4EEEEEEllnn$$$$$$$$fllnn$$$$$$$$ "!)DtTTTTTTTTTF"19UQUVVVVVVVVVG&'')) D D92244CCCCCD D D D D D D D D D D D D D Dy..00FJ>>>>>llnn$$$$$$$mmoo%%%%%%%% llnn$$$$$$$mmoo%%%%%%%%9**,,0AAAAAAS)	B )	B )	B )	B )	B )	B )	B )	B )	B )	B )	B )	B )	B )	B )	BV y&&((DJ,?????ai4PPPPPPPPP	!9**,,
::::955 C Cy..00DM4EEEEE&&(( C C92244
BBBBBC C C C C C C C C C C C C C CC C C C C C C C C C C C C C C
 ,,..        &,,..         o7! 7! 7! 7! 7! 7! 7! 7! 7! 7! 7! 7! 7! 7! 7! 7! 7! 7!sD  AW30AQ%3D6Q%D-- Q%)F8Q%F//AQ%AK
,J7KJK
J6KQ%K99=Q%7O,*N6O,NO,	N
,O,75Q%,6P""6Q%W3%Q)	)W3,Q)	-AW3A	W
AV *V	=V 	VV VV W V$	$W'V$	(W+W3W""W33W7:W7c                   K   t          j        dd          }|                    d           }t          d |j        D                       rJ d S )Nr  r4   rk   c                    | dz  S )N   r   r  s    r*   r   z*test_bag_groupby_default.<locals>.<lambda>  s
    QV r,   c              3  *   K   | ]}d |d         v V  dS )partdr   Nr   )r   ks     r*   r  z+test_bag_groupby_default.<locals>.<genexpr>  s*      44q7ad?444444r,   )r   ro   r   r  r^   )r7   r8   r9   r:   b2s        r*   test_bag_groupby_defaultr?    sZ      
"%%%A	
##	$	$B44BG4444444444r,   )q
__future__r   rV   rW   r   r   rG  sys	functoolsr   operatorr   r   r   r	   distributed.utils_testr
   r   r7   r   r   r   r   r   r   r   r^   dask.bagbagr   r   r   r   	dask.baser   r   dask.blockwiser   dask.delayedr   r)   r   r   dask.layersr   r   
dask.utilsr   r   r    r!   dask.utils_testr"   markskipifplatform
pytestmarkfilterwarningsignore_sync_scheduler_warningr+   r1   r@   rN   rh   rr   parametrizer   r   r   r   r   r   xfail_with_pyarrow_stringsr   paramxfailr   r   r   r   r  r  r  r  r&  r0  skip_with_pyarrow_stringsflakyrR  rW  ri  rl  rx  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r&  r$  r%  r+  r7  r?  r   r,   r*   <module>rY     s
   " " " " " " !f!-00  				 



             = = = = = = = = = = * * * * * * . . . . . .                         * * * * * * * * * * = = = = = = = = $ $ $ $ $ $             - - - - - - - - 8 8 8 8 8 8 8 8 J J J J J J J J J J J J      >>+6666'+%@@@Kgg%888G [LG	(    
 !' : :2! ! ( ( (0 0 0 D0 0 0  && & &"	- 	- 	- $//  0/4 cU|,,tUm44  54 -,&       )O )O   )OX  ; ; ; D    '  (' 	+##+GPU#VV	
 	
 	
 	+##+GPU#VV	
 	
 	

  '. . ('   . D  "	S 	S 	S	% 	% 	%	 	 	R R R D0 0 0   D$ $ $<     !4!4!455. . 65 .6 !)N	
 	
 	
 	U&+"3"31"3"="=>>>  !4!4!455$77'1 '1 87 65  '1T. . .& D    D  / / /, D! ! !H$ $ $2 D	2 	2 	2 DW W W. %	 	
L  DG9q=111"8 "8 21 "8J D> > >"' ' 'T 2  D
 
 	 

. D0 0 00@ @ @/ / /	% 	% 	%, , , !   2 2 2    : : :  &k.>.C.C.E.E'FGG9 9 HG '&9 [&)B777:! :! 87:!z D5 5 5 5 5r,   