
    tf                    ,   d dl mZ d dlZ ej                  d      Zd dlZd dlZd dlZd dlm	Z	 d dl
mZ d dlmZmZmZ d dlmZ d dlmZ d d	lmZmZmZmZmZmZmZ d dlZd dlmZ d d
lmZmZm Z  d dl!m"Z"m#Z# d dl$m%Z% d dl&m'Z' d dl(m)Z)m*Z* d dl+m,Z,m-Z- d dl.m/Z/m0Z0m1Z1m2Z2 d dl3m4Z4 d e/e      v r e	ed      Z e	ed      Zejj                  jm                  ejn                  dk(  d      Z8ejj                  js                  d      Z:d Z;d Z< ed      d        Z=d Z>d Z?d Z@ejj                  j                  d ddg      d!        ZBejj                  j                  d"d#d#gg      ejj                  j                  d$ddg      d%               ZCejj                  j                  d&g d'      ejj                  j                  d(d)d*g      d+               ZDd, ZEd- ZFe: ed      d.               ZGejj                  j                  d/        ZIe:ejj                  j                  d0d1 ej                  d2ejj                  j                  d3d4      5       ej                  d6ejj                  j                  d3d4      5      g      ejj                  j                  d7                      ZL ed      d8        ZMd9 ZNd: ZOd; ZPd< ZQd= ZRd> ZS ed      d?        ZTd@ ZU ed      dA        ZVejj                  j                  dBg dC      ejj                  j                  d g dD      dE               ZWe:ejj                  j                  dBdF ej                  dGejj                  j                  5      dH ej                  dIejj                  j                  dJK      5      g      ejj                  j                  d g dD      ejj                  j                  dLddg      dM                             ZZdN Z[ ed      dO        Z\ ed      dP        Z]dQ Z^ ed      dR        Z_dS Z` ed      dT        Za ed      dU        Zbejj                  j                  dVdWe-fdXe,fg       eddYgdXz  Z      d[               Zc ed      d\        Zdd] Zeejj                  js                  d       ed      d^               Zf ed      d_        Zgd` Zhda Zidb Zjdc Zkejj                  j                  ddg de      df        Zlejj                  j                  dgg dh      di        Zmejj                  j                  ejj                  j                  djdej                  j                         g      dk               Zp edldmig n      do        Zq ed      dp        Zry)q    )annotationsNdistributed)partial)add)ClientSchedulerPluginWorkerPlugin)cleanup)client)clustercluster_fixturegen_clusterlooploop_in_threadpopenvarying)computedelayedpersist)compute_as_if_collectionget_scheduler)	Blockwise)Delayed)
futures_ofwait)ShuffleLayerSimpleShuffleLayer)get_named_argsget_scheduler_locktmpdirtmpfile)incshould_check_stateF)r#   win32zTThe teardown of distributed.utils_test.cluster_fixture fails on windows CI currentlyreasonzrignore:Running on a single-machine scheduler when a distributed client is active might lead to unexpected results.c                     ddl m}  y )Nr   r   )dask.distributedr   r(   s    e/var/www/html/software/conda/envs/higlass/lib/python3.12/site-packages/dask/tests/test_distributed.pytest_can_import_clientr+   >   s    '    c                     ddl m}  y )Nr   dumps)dask.distributed.protocolr/   r.   s    r*   test_can_import_nested_thingsr1   B   s    /r,   Tc                  K    t        t              d      }t        |      \  }t        |       d {    |j                  |j
                  v s|j                  |j
                  v sJ  t        t              d      }t        |d      \  }}t        |       d {    |j                  |j
                  v s|j                  |j
                  v sJ y y 7 7 :w)N   
   )r   r"   r   r   keydata)	csabxx2yy2ones	            r*   test_persistr@   F   s     QAAJER
r(NN66QVVrvv///RAamGB
r(NN66QVVrvv////  s"   1CCA(CC9CCc                   t        d      dz   }|dz   }|dz   } t        |dd|gd| dfddg      }t        |d   d   t              sJ t        |d   d   d   t              sJ t        |d   d   t              sJ d	g d
ddddgf}t	        | |k(  sJ t        ||g| ddgd      }|d   d   |u sJ |d   d   |u sJ |d   j	                         dk(  sJ |dd  ddgfk(  sJ y )Nr3         r9   r:      r   r9   r:      )r3   rC      )   rC   F)traverserH   )r   r   
isinstancer   r   )r7   r9   r:   resultsolress         r*   test_persist_nestedrN   U   s*   
QA	AA	AA1Aq!9-1vq1#>FfQing...fQinQ'111fQilG,,,#VQ
4CFs"""
1a&!Qe
4Cq6!9>>q6!9>>q6>>q   qr7q1#hr,   c                N   t        j                  d      }t        j                  d      }|j                  dg di      } | j                  ||g      }|j	                  |      }|j
                  j                  |j                         |j                  ||gd             |j                         s3t        |j                  j                  |j                     t              sJ t        j                  t               5  |j	                  ddg      }d d d        y # 1 sw Y   y xY w)	Npandasdask.dataframer;   r3   rC      r   axisr3   rC   )pytestimportorskip	DataFramescatterfrom_delayedutils	assert_eqr   concat_dask_expr_enabledrJ   dasklayers_namer   raises	TypeError)r7   pddddffuturesddfs         r*   !test_futures_to_delayed_dataframeri   h   s    			X	&B			-	.B	sI&	'BaiiR!G
//'
"CHHs{{}biiRqi&AB   "#((//#))4i@@@	y	! &ooq!f%& & &s   >DD$c                0   t        j                  d      }t        j                  d      }|j                  dt        d      i      }|j	                  |d      }|j                  |j                               }|j                  j                  |||        y )NrP   rQ   r;      rC   npartitions	scheduler)	rV   rW   rX   rangefrom_pandasrZ   
to_delayedr[   r\   )r7   rd   re   rf   rh   s        r*   test_from_delayed_dataframers   z   s|     
		X	&B			-	.B	sE"I&	'B
...
+C
//#..*
+CHHsB!,r,   fusec                   t        j                  d      }t        j                  d      }d}|j                  t        |      t        |      d      }|j                  t        |      t        |      d      }|j	                  ||      dz   }|j	                  |d      dz   }|dz  }|dz  }t
        j                  j                  d	|i      5  |j                  |d
gdd      }	|	j                          |	j                         j                  d
      }
d d d        |j                  j                  
|j                  |d
gd      j                  d
      d       y # 1 sw Y   JxY w)NrP   rQ   #   r;   r=   )r;   zrl   r4   rB   optimization.fuse.activer;   lefttasks)onhowshuffle_method)r|   r}   Fcheck_index)rV   rW   rX   rp   rq   r_   configsetmergeheadr   sort_valuesr[   r\   )r7   rt   rd   re   sizedf1df2ddf1ddf2ddfmdfms              r*   $test_fused_blockwise_dataframe_merger      s?   			X	&B			-	.B
 D
,,U4[uT{;
<C
,,U4[uT{;
<C>>#4>025D>>#1>-2D2IC2IC	4d;	< .zz$C5fWzM		lln((-. HHSYYsu&Y1==cBPU  . .s   AEE(r|   r9   	broadcastc                   t        j                  d      }t        j                  d      }|j                  ddgdz  t        d      d      }|j                  ddgt        d      d      }|j	                  |d      }|j	                  |d      }|j                  ||||d	
      }	|	j                         }
|j                  j                  |
j                  d      |j                  |||      j                  d      d       y )NrP   rQ   r3   rC   rE   )r9   b_left)r9   b_rightrl   r{   )r|   r   r~   r9   )r|   Fr   )
rV   rW   rX   rp   rq   r   r   r[   r\   r   )r7   r|   r   rd   re   pdflpdfrdfldfrr   r   s              r*   test_dataframe_broadcast_merger      s     
		X	&B			-	.B<<q!fqjE!H=>D<<q!fq:;D
..1.
-C
..1.
-C88Cy8QD
,,.CHH
t#//4  r,   computation)Nr   dask.computezscheduler, use_distributed)NT)syncFc                L  	 t        j                  d      	t        j                  d       G d dt              } | j                   |              	fd} | j                  |      }|j                         du sJ  | j                  d       }|r|dk(  sJ |       y	y	)
zvShould a collection use its default scheduler or the distributed
    scheduler when being computed within a task?
    rP   rQ   c                      e Zd Zd Zd Zy)<test_default_scheduler_on_worker.<locals>.UpdateGraphCounterc                   K   d|_         y w)Nr   _update_graph_count)selfro   s     r*   startzBtest_default_scheduler_on_worker.<locals>.UpdateGraphCounter.start   s     ,-I)s   	c                .    |xj                   dz  c_         y Nr3   r   )r   ro   argskwargss       r*   update_graphzItest_default_scheduler_on_worker.<locals>.UpdateGraphCounter.update_graph   s    ))Q.)r,   N)__name__
__module____qualname__r   r    r,   r*   UpdateGraphCounterr      s    	.	/r,   r   c                 Z   d} j                  t        |       t        |       d      }j                  |d      }|j                         ydk(  rt	        j                  |       ydk(  r7t        |j                  |j                  t        |j                               yJ )	Nr4   rw   rC   rl   rn   r   r   T)rX   rp   rq   r   r_   r   	__class__list)r   rf   rh   r   re   rd   ro   s      r*   fooz-test_default_scheduler_on_worker.<locals>.foo   s    \\d%+>?nnRQn/KK)K,  N*LL	2  66$sxxchh9  5r,   Tc                    | j                   S Nr   )dask_schedulers    r*   <lambda>z2test_default_scheduler_on_worker.<locals>.<lambda>   s    ~AA r,   rC   N)rV   rW   r   register_pluginsubmitrK   run_on_scheduler)
r7   r   use_distributedro   r   r   rM   num_update_graphsre   rd   s
    ` `    @@r*    test_default_scheduler_on_workerr      s    * 
		X	&B			-	.B/_ / A(*+" !((3-C::<4***A &5!N=NN;!!r,   c                    g d} | j                   ||g      }t        j                  |      }t        |      ||z   k(  sJ y )NrR   )rY   dbrZ   r   )r7   Lrg   r:   s       r*   test_futures_to_delayed_bagr      s?    AaiiAG
 A7a!er,   c           
        t        j                  d       t        j                  d      }ddlm} t        j                  d      }|j	                  d      } | j
                  ||g      }|j                  |D cg c]*  }|j                  ||j                  |j                        , c}d      } ||j                         |j                  ||gd             y c c}w )Nnumpy
dask.arrayr   r\   rB   )shapedtyperT   )rV   rW   dask.array.utilsr\   arangerY   concatenaterZ   r   r   r   )r7   dar\   npr;   rg   fAs           r*   test_futures_to_delayed_arrayr      s    
 			\	*B*			W	%B
		!AaiiAG
CJKa!''	9KRS 	 	A aiik2>>1a&q>9: 	Ls   :/Cc                  K   t         j                  j                  d      5   t        t              d      j                         }d d d        t        j                  d       d {    |j                  rJ  t        t              d      j                  d      }t        j                  d       d {    |j                  rJ y # 1 sw Y   xY w7 o7 !w)Nr   rn   r3   g{Gz?rC   )	r_   r   r   r   r"   r   asynciosleepr{   r7   r8   r9   r:   r;   s        r*   &test_local_get_with_distributed_activer   	  s      
6	* &GCLO##%&
--
ww;Q&1A
--
ww;w& & s;   !C$C C'C(AC7C8CCCCc                r    t        j                  d       t        j                  d       ddlm}  |        y )Nr   rP   r   )test_to_hdf)rV   rW    dask.dataframe.io.tests.test_hdfr   )r7   r   s     r*   test_to_hdf_distributedr     s&    
 
!<Mr,   rm   r3   rE   zHDF not multi-process safe)r&   strict)marksr4   c                v    t        j                  d       t        j                  d       ddlm}  |d |        y )Nr   rP   r   )test_to_hdf_schedulers)rV   rW   r   r   )rm   r7   r   s      r*   !test_to_hdf_scheduler_distributedr      s-    "  
!G4-r,   c                F  K   t        j                  d      }t        j                  d      }|j                  g dg dd      }|j                  |d      }|j	                  dd	
      j                  dd      } | j                  |j                  d          d {   }	 | j                  |j                  d          d {   }
|j                  j                  |j                  |	|
g      |j                  ddgddgd      j                  d             y 7 ~7 Yw)NrP   rQ   )r3   rC   rS   rE   )r3   r   r3   r   rw   rC   rl   r=   F)sortcount)	split_outr   r3   )rV   rW   rX   rq   groupbyaggr   
partitionsr[   r\   r]   	set_index)r7   r8   r9   r:   rd   re   rf   rh   rK   agg0agg1s              r*   test_serializable_groupby_aggr   9  s     			X	&B			-	.B	L|<	=B
...
+C[[5[)--g-CF 6,,Q/00D6,,Q/00DHH
		4,
Aq6A/0::3? 10s%   BD!D&D!DAD!D!c                2   t        d      t        d      }} t        t              ||      } t        t              ||      } t        t              ||      } | j                  |      } t        t              |d      }|j                  d      dk(  sJ y )Nr3   rC   r4   dask.distributedrn      )r   r   r   r   )r7   r;   r=   xxyyxxyyxxyy2xxyy3s           r*   test_futures_in_graphr   K  s    1:wqzqA	a	B	a	B73<BDAIIdOEGCL#E==#5=6:RRRRr,   c                   t        j                  d       t        j                  d      }t        j                  d       t               5 }|j                  dd      }|j	                  |       |j                  |      }|j                  |||        |j                  |j                  k(  sJ 	 d d d        y # 1 sw Y   y xY w)Nr   r   zarrrS   rS   r3   r3   chunksrn   )rV   rW   r    zerosto_zarr	from_zarrr\   r   )r7   r   dr9   a2s        r*   test_zarr_distributed_roundtripr   W  s    
 			\	*B
	 %QHHVFH+			!\\!_
Qa(yyAHH$$$% % %s   
A%B99Cc                ,   t        j                  d       t        j                  d      }t        j                  d      }t               5 }d}|j                  d|      }|j                  j                  |      }|j                  j                  |j                  ||j                  |d      }|j                  |       |j                  |      }|j                  ||| 	       |j                  |j                  k(  sJ 	 d d d        y # 1 sw Y   y xY w)
Nr   r   r   r   r   r   r9   r   r   r   storemodern   )rV   rW   r    r   storageDirectoryStorecreation
open_arrayr   r   r   r   r\   r   )	r7   r   r   r   r   r9   r8   rx   r   s	            r*   3test_zarr_distributed_with_explicit_directory_storer  d  s    
 			\	*Bv&D	 %QHHVFH+LL''*MM$$'''' % 
 	
		!\\!_
Qa(yyAHH$$$% % %s   
B6D

Dc                   t        j                  d       t        j                  d      }t        j                  d      }d}|j                  d|      }|j                  j	                         }|j
                  j                  |j                  ||j                  |d      }t        j                  t        d	
      5  |j                  |       d d d        y # 1 sw Y   y xY w)Nr   r   r   r   r   r   r9   r   distributed schedulermatch)rV   rW   r   r   MemoryStorer   r  r   r   rb   RuntimeErrorr   )r7   r   r   r   r9   r8   rx   s          r*   0test_zarr_distributed_with_explicit_memory_storer	  z  s    
 			\	*Bv&DF
'A  "A  gggg 	! 	A 
|+B	C 			!  s   >CC"c                T   t        j                  d       t        j                  d      }t        j                  d      }d}|j                  d|      }|j                  ||      }t        j                  t
        d      5  |j                  |       d d d        y # 1 sw Y   y xY w)	Nr   r   r   r   r   r   r  r  )rV   rW   ones
zeros_likerb   r  r   )r7   r   r   r   r9   rx   s         r*   #test_zarr_in_memory_distributed_errr    s    
 			\	*Bv&DF
v&A&)A	|+B	C 			!  s   BB'c                     t        d              j                  |       dk(  sJ  | j                  fd      sJ y )Nc                      yr   r   r   r,   r*   r   z.test_scheduler_equals_client.<locals>.<lambda>  s    r,   rn   r3   c                :    | j                  j                        S r   )storyr5   )r   r;   s    r*   r   z.test_scheduler_equals_client.<locals>.<lambda>  s    ^5I5I!%%5P r,   )r   r   r   )r7   r;   s    @r*   test_scheduler_equals_clientr    sB    	A99q9!Q&&&1PQQQr,   c                   K    t        j                  t              d      }|j                          d {   }|j                  |j
                  v sJ |j                  s|j                  sJ t        d t        |      D              sJ y 7 Vw)Nr3   c              3  <   K   | ]  }|j                           y wr   )done).0r   s     r*   	<genexpr>ztest_await.<locals>.<genexpr>  s     /Aqvvx/s   )	r_   r   r"   r   r5   r{   r6   allr   r   s        r*   
test_awaitr    so     S!AiikA55AGG66QVV/A//// 	s   3BBABc                 >    d } t        j                   |               y )Nc                    K    t        j                  t              d      } | dz   }|j                          d {   }t	        |j                         dk(  sJ y 7 wr   )r_   r   r"   r   len)r;   r=   rx   s      r*   r   ztest_local_scheduler.<locals>.f  sM     DLLa E))+166{a s   8AA A)r   run)r   s    r*   test_local_schedulerr    s      KKr,   c                L  K   t        j                  d       t        j                  d      }t        j                  d      }ddlm} t	        t        d      t        d      ddg      fd}d	 }|j                  d
d      }	t        j                  d      5  |	j                  ||j                  d|j                              }
d d d        t        j                  d      5  
j                  ||j                  d|j                              }d d d        t        j                  j                  d      5   | j                         d {   }d d d         ||j                  d
      dz         y # 1 sw Y   xY w# 1 sw Y   txY w7 @# 1 sw Y   ?xY ww)Nr   r   r   r   r?   tworC   c                            | z  S r   r   )r;   scales    r*   flaky_doublez7test_annotations_blockwise_unpack.<locals>.flaky_double  s    w{r,   c                    d| z  S )NrC   r   r;   s    r*   reliable_doublez:test_annotations_blockwise_unpack.<locals>.reliable_double  s    1ur,   r4   )rB   r   )retriesr   r   metaF)optimization__fuse__activeg      @)rV   rW   r   r\   r   ZeroDivisionErrorr  r_   annotate
map_blocksarrayfloat64r   r   r   )r7   r8   r9   r:   r   r   r\   r#  r&  r;   r=   rx   r"  s               @r*   !test_annotations_blockwise_unpackr1    sb    
 			\	*B			W	%B* &u-/@/GANOE 	4 A 
q	! LLLBHHRrzzH,JLKL	q	! OLLrxx"**x/MLNO 
E	: !))A, as"#L LO O  s`   BF$/E>
F$(/F
(F$?FFF$F$>FF$
FF$FF!F$io)r  r   full)TFNc                   t        j                  d      }t        j                  d      }d}d}|dk(  r%|j                  ||      }|j                  |      }nU|dk(  r%|j                  ||      }|j                  |      }n+|dk(  r&|j	                  |d	|      }|j	                  |d	      }d
z  }d
z  }t
        j                  j                  d|i      5  |j                          t
        j                  j                  |j
                  |j                               }	t        |	t              |duk(  sJ |j                  |||        d d d        y # 1 sw Y   y xY w)Nr   r   )rB   rC   )r4   rE   r  r   r   r3  r4   rC   ry   Frn   )rV   rW   r  r   r3  r_   r   r   r   r/  optimize__dask_keys__rJ   dictr\   )
r7   r2  rt   r   r   r   r   darrnarrdsks
             r*   test_blockwise_array_creationr;    s:    
		W	%B			\	*BFE	V|wwuVw,wwu~	wxxfx-xx	vwwubw0wwub!AIDAID	4d;	< .jj!!$))T-?-?-AB#t$U):;;;
T41-. . .s   A4EEparquet-pyarrowparquet-fastparquetcsvhdfrB   )rerunsfrom_futuresc                   t        j                  d      }t        j                  d      }|j                         rt        j                  d       |j	                  g ddz  t        d      d      }|rU|j                  d d |j                  dd |j                  dd g} | j                  |      }	|j                  |	|d	   
      }
n|j                  |d      }
|dk(  rKt        j                  d       |
j                  t        |             |j                  t        |            }n4|dk(  rpt        j                  d       t        j                  t              5  |
j                  t        |      d       |j                  t        |      d      }d d d        n|dk(  rU|
j                  t        |      d       |j!                  t"        j$                  j'                  t        |      d            }ne|dk(  rUt        j                  d       t         |j&                  d            }|
j)                  |d       |j+                  |d      }nt-        d      |dg   dz   }dg   dz   }|j                         st.        j0                  j3                  d|i      5  |j5                          t.        j6                  j9                  |j.                  |j;                               }t=        |t>              tA        |      k(  sJ |jC                  ||d       d d d        y y # 1 sw Y   xY w# 1 sw Y   y xY w)NrP   rQ   zdoesn't work yetrR   rB      rw   r4   r   r)  rS   rl   r<  pyarrowr=  fastparquet)enginer>  F)index*r?  tablesh5z/data*unreachabler;   ry   r   )"rV   rW   r^   xfailrX   rp   ilocrY   rZ   rq   
to_parquetstrread_parquetwarnsFutureWarningto_csvread_csvospathjointo_hdfread_hdfAssertionErrorr_   r   r   r   	dataframer5  r6  rJ   r7  boolr\   )r7   r    r2  rt   rA  rd   re   rf   partsfutsddf0rh   fnr:  s                 r*   test_blockwise_dataframe_iora    s     
		X	&B			-	.B	'(	IMb	:	;B!bggamRWWR^<qyyt%(3~~ba~0	I&F$ooc&k*	$	$M*\\-( 	EOOCKO>//#f+m/DC	E 	E 
uCKu-kk"'',,s6{C89	uH%T"#B!kk"h']++	SERB
se*r/C  "[[__8$?@ 	5KKM..))#((C4E4E4GHCc4(DJ666LLbeL4	5 	5 #!	E 	E"	5 	5s   9L(#A;L4(L14L=c                B   t        j                  d      }t        j                  d      }|j                  dg ddz  i      }|j                  |d      d   }|dk  }t	        |      }|d	k(  sJ |j
                  |j                            j                         d	k(  sJ y )
NrP   rQ   r;   rR   rB   rC   rl   rS   rC  )rV   rW   rX   rq   r  r;   r   sum)r7   rd   re   rf   seriesrK   
series_lens          r*   #test_blockwise_fusion_after_computerf  5  s     
		X	&B			-	.B 
sIM*	+B^^BA^.s3FaZF
 VJ44 !%%'2---r,   c           	     |  K   t        j                  d       t        j                  d      }t        j                  d      fd}|j                  |d|j                  d      dj	                  d      dj                        } | j
                  |j                         d	
       d{   }|dk(  sJ y7 w)zDTest pack/unpack of blockwise that includes a NumPy literal argumentr   r   c                X    t        |      j                  u sJ | j                  |      S r   typeuint16astyper;   dtr   s     r*   r`  z%test_blockwise_numpy_args.<locals>.fnO  '    Bx299$$$xx|r,   r;     *   Nr(  Foptimize_graphrV   rW   	blockwiser  rk  r   rc  	r7   r8   r9   r:   r   r`  arrrM   r   s	           @r*   test_blockwise_numpy_argsrx  H  s       			\	*B			W	%B ,,
CRYYr]D		  C 		#'')E:
:C$;; ;   B*B<-B:.B<c           
     |  K   t        j                  d       t        j                  d      }t        j                  d      dfd	}|j                  |d|j                  d      dj                  j	                  d            } | j
                  |j                         d	
       d{   }|dk(  sJ y7 w)zLTest pack/unpack of blockwise that includes a NumPy literal keyword argumentr   r   Nc                X    t        |      j                  u sJ | j                  |      S r   ri  rm  s     r*   r`  z'test_blockwise_numpy_kwargs.<locals>.fna  ro  r,   r;   rp  rq  )r   rn  Frr  r   rt  rv  s	           @r*   test_blockwise_numpy_kwargsr|  Z  s       			\	*B			W	%B ,,r3s"))		RT,
VC		#'')E:
:C$;; ;ry  c                   t        j                  d      }t        j                  d      }|j                  |j                  d            }|j                  |j	                  g d            }|j                         }||z  }t        j                  |      \  }||z  }|j	                  g d      }t        j                  j                  ddi      5  |j                         }	|j                         }
d d d        |j                  j                  	|       |j                  j                  
|       y # 1 sw Y   BxY w)Nr   r   rS   )y      $@       @y      @      y       @      ?)y                y      @      @y      0@       ry   F)rV   rW   
from_arrayr   r/  conjr_   r5  r   r   r   testingassert_equal)r7   r   r   uvcvr;   r=   expectedx_valuey_values              r*   %test_blockwise_different_optimizationr  j  s    
 
		W	%B			\	*B
biil#A
bhh89:A	
B	BAMM"ER	BAxx12H	4e<	= ))+))+ JJGX.JJGX.	 s   !D44D=c                  K   t        j                  d      }t        j                  d      }t        j                  d      }t        j                  d      }d } | j                  d d      }	 | j                  d d	      }
|j                  |d
|j	                  dd      d
|j                  dd      d
|	ddt        |
      }|j                  |j                  d|j                  d	      i      d	      }|j                  dd      }|d   j                         }|j                         |j                         z   } | j                  |d       d{   }|dk(  sJ y7 w)z9Check pack/unpack of a HLG that has every type of Layers!r   rP   r   rQ   c                    | |z   |z   |z   S r   r   )r;   r=   rx   	extra_args       r*   r   z&test_combo_of_layer_types.<locals>.add  s    1uqy9$$r,   c                    | S r   r   r%  s    r*   r   z+test_combo_of_layer_types.<locals>.<lambda>      1 r,   rC   c                    | S r   r   r%  s    r*   r   z+test_combo_of_layer_types.<locals>.<lambda>  r  r,   rS   r;   )rS   r3   r   NF)r   r   r  r9   rl   r{   )r~   rr     )rV   rW   r   ru  r   r  intrq   rX   r   shuffleto_dask_arrayrc  r   )r7   r8   r9   r:   r   rd   r   re   r   r=   rx   r;   rf   rM   s                 r*   test_combo_of_layer_typesr    sL     
		W	%B			X	&B			\	*B			-	.B% 	a Aa A

d#
T"	 	 	A 
c299Q<%89q	IB	C	0B	C			 B
%%'BFFH
C		#e4
4C"99 5s   EE$E"E$c           
        t        j                  d      }t        j                  d      fd}j                  |j                  g d            }j                  |j                  g dg            }j	                  |d|d|d|j
                  d      } | j                  |d	
       j                  |||        y)z1Test a blockwise operation with concatenated axesr   r   c                2    j                  |g dg       | S )Nr   r3   rC   r   )r;   r=   r   s     r*   r   z%test_blockwise_concatenate.<locals>.f  s    
Q$r,   r  iijT)r   r   Frr  rn   N)rV   rW   r~  r/  ru  r   r   r\   )r7   r   r   r;   r=   rx   r   s         @r*   test_blockwise_concatenater    s    			W	%B			\	*B 	bhhy)*A
bhh	{+,A
						gg 	 		A AIIa&LLAL#r,   c                H  K   t        j                  d      }t        j                  d      }|j                  |j                  dt	        d      i      d      } | j
                  |j                  dd             d {   }|d   ddd	k(  sJ |d
   d
dd	k(  sJ y 7 w)NrP   rQ   r9   r4   rC   rl   c                    |S r   r   )r;   partition_infos     r*   r   z4test_map_partitions_partition_info.<locals>.<lambda>  s    . r,   r   )numberdivisionr3   rB   r   )rV   rW   rq   rX   rp   r   map_partitions)r7   r8   r9   r:   rd   re   rh   rM   s           r*   "test_map_partitions_partition_infor    s     			X	&B			-	.B
..sE"I&67Q.
GC		HI C q6q1111q6q1111	s   A>B" B  B"c                  K   t        j                  d      }t        j                  d      }|j                  |j                  t	        t        d      |j                  dddd                  d	
      }||j                  j                  t        d               j                         }|j                  j                  j                         |d<    | j                  |j                  j                  |dgd       d{   }y7 w)z.Copied from distributed (tests/test_client.py)rP   rQ   2   z
2020-01-01z
2020-09-01UTC)r   endperiodstz)uid
enter_timer3   rl      dayF)columnsrG  N)rV   rW   rq   rX   r7  rp   
date_ranger  isinr   r  rn  day_namer   categorical
categorize)r7   r8   r9   r:   rd   re   rh   s          r*   test_futures_in_subgraphsr    s      
		X	&B			-	.B
..
"I==&L" ) 	
   
C cggll59%
&
.
.
0C""++-CJ22C%PUV
VC
Vs   C>D DDzmax_branch, expected_layer_type    rC   ) r3   )r   nthreadsc                  K   t        j                  d      }t        j                  d      }|j                         rt        j                  d        G fddt               | j
                                 d {    |j                  dt        d      i      }|j                  |d	      }	|	j                  dd
|      }
t        |
j                  j                        t        |	j                  j                        z
  }|D ],  }d|v st        |
j                  j                  |   |      r,J   | j                  |
       d {    j                  rJ y 7 7 w)NrP   rQ   z"Checking layers doesn't make sensec                  "    e Zd ZdZd Z fdZy)?test_shuffle_priority.<locals>.EnsureSplitsRunImmediatelyPluginFc                    || _         y r   )worker)r   r  s     r*   setupzEtest_shuffle_priority.<locals>.EnsureSplitsRunImmediatelyPlugin.setup  s	     DKr,   c                
   |dk(  r}t        d | j                  j                  j                  D              sLt	        d t        | j                  j                  j                        D              rd_        t        d      y y y )N	executingc              3  8   K   | ]  }d |j                   v   ywsplitNr5   r  tss     r*   r  z]test_shuffle_priority.<locals>.EnsureSplitsRunImmediatelyPlugin.transition.<locals>.<genexpr>  s      1&(266!1   c              3  8   K   | ]  }d |j                   v   ywr  r  r  s     r*   r  z]test_shuffle_priority.<locals>.EnsureSplitsRunImmediatelyPlugin.transition.<locals>.<genexpr>  s     QRw"&&(Qr  TzSplit tasks are not prioritized)	r  r  stater  anyr   readyfailurer  )r   r5   r   finishr    EnsureSplitsRunImmediatelyPlugins        r*   
transitionzJtest_shuffle_priority.<locals>.EnsureSplitsRunImmediatelyPlugin.transition  sw    $S 1,0KK,=,=,G,G1 . Q48I8I8O8O3PQQ?C4<&'HII R.$r,   N)r   r   r   r  r  r  )r  s   r*   r  r    s    	!	Jr,   r  r9   rp  r4   rl   r{   )r~   
max_branchr  )rV   rW   r^   skipr	   r   rX   rp   rq   r  r   r_   r`   rJ   r   r  )r7   r8   r9   r:   r  expected_layer_typerd   re   rf   rh   r   shuffle_layers
layer_namer  s                @r*   test_shuffle_priorityr    s>     
		X	&B			-	.B	89J< J !

<>
??? 
sE$K(	)B
...
,C;;s7z;JD))*S-AAN$ Q

"dii..z:<OPPPQ !))D//77777# @  s1   A;E#>E?BE##E#3E#E!E#!E#c                *  
K   t        j                  d      
t        j                  d      t        j                  d      }t        j                  d      }t        j                  d      }|j                         rt        j                  d       
fd}|j	                  d	      j                         }|j                  d
d      j                         }	 | j                  |j                  ||	|j                               d{    y7 w)z7Check that map_partitions can handle a dask array inputr   rP   r   rQ   zdask.datasetsz-roundtripping through arrays doesn't work yetc                h    t        | j                        sJ t        |j                        sJ | S r   )rJ   rX   ndarray)r   r9   r   rd   s     r*   r   z'test_map_partitions_da_input.<locals>.f  s0    !R\\***!RZZ(((r,   1d)freqr  r3   r   r)  N)
rV   rW   r^   r  
timeseriesr   r  r   r  _meta)r7   r8   r9   r:   r   re   datasetsr   rf   rw  r   rd   s             @@r*   test_map_partitions_da_inputr    s      
		W	%B			X	&B			\	*B			-	.B""?3H	CD
 
		$		'	/	/	1B
''$q'
!
)
)
+C
!))B%%a288%<
===s   DDDDc            	        t        j                  d      t        j                  d      j                         rt        j                  d       fdfd} t        j                  dddd	id	d
d
d	      5 }t        j                  |d	      5   |         ddd       ddd       y# 1 sw Y   xY w# 1 sw Y   yxY w)z[
    Check that map_partitions can handle a delayed
    partition of a dataframe input
    rP   rQ   z/map partitions can't deal with delayed properlyc                h    t        | j                        sJ t        |j                        sJ | S r   )rJ   rX   )r   r9   rd   s     r*   r   z'test_map_partitions_df_input.<locals>.f0  s0    !R\\***!R\\***r,   c                 h   j                  j                  dt        d      i      d      } | j                         d   j	                         }j                  j                  dt        d      i      d      }|j                  dd      }|j                  ||d	
      j                          y )Nr9   r4   r3   rl   r   r:   r{   )r|   r~   F)r*  enforce_metadata)rq   rX   rp   rr   r   r  r  r   )item_dfrh   	merged_dfre   r   rd   s      r*   mainz*test_map_partitions_df_input.<locals>.main5  s    ..sE"I.>!?Q.O  "1%--/NN2<<eBi0@#AqNQ	 %%W%E	  sU 	! 	

')r,   r   z:0	dashboardFr3   )scheduler_portdashboard_addressscheduler_kwargsasynchronous	n_workersr  	processes)r  N)rV   rW   r^   r  r   LocalClusterr   )r  r   re   r   rd   s     @@@r*   test_map_partitions_df_inputr  &  s    
 
		X	&B			-	.B	EF

 
	!	!
 %u- 
" 
  
e< 	F	 	 	 s$   =B:B.B:.B7	3B::Cc           	     ,  K   t        j                  d      }t        j                  d      }t        j                  d       |j                  t        d      t        d      d      }d|j                  _        |j                  |d      }t               5 }d	| }	|j                  d
|	dddid      }
 | j                  |
       d {    |j                  j                  ||j                  d
|	d      d       d d d        y 7 <# 1 sw Y   y xY ww)NrP   rQ   
sqlalchemyr4   rD   rG  r3   rl   z
sqlite:///testTechoF)rG  engine_kwargsr   )check_divisions)rV   rW   rX   rp   rG  namerq   r!   to_sqlr   r[   r\   read_sql_table)r7   r8   r9   r:   rd   re   rf   rh   r   urirK   s              r*   test_to_sql_engine_kwargsr  R  s     
		X	&B			-	.B
%	E"IE"I6	7BBHHM
...
+C	 
a1#CtFE?E  
 aii
fc73! 	 	

 

 	 
 
s0   BD3D	D
3D=	DDDDc                  
K   t        j                  d      }t        j                  d      }|j                         rt        j                  d        G d d      
d}|j	                  dg|z  t        d|dz         	      }|j                  |d
      }|j                  d 
fdddt              }	 | j                  |	       d {   j                  dk(  sJ y 7 w)NrP   rQ   z!we don't offer a public reductionc                      e Zd Zd Zy)0test_non_recursive_df_reduce.<locals>.SomeObjectc                    || _         y r   )val)r   r  s     r*   __init__z9test_non_recursive_df_reduce.<locals>.SomeObject.__init__w  s	    DHr,   N)r   r   r   r  r   r,   r*   
SomeObjectr  v  s    	r,   r     r3   rC   )r6   rG  "   rl   c                    | S r   r   r%  s    r*   r   z.test_non_recursive_df_reduce.<locals>.<lambda>~  s     r,   c                L     | j                         j                               S r   )rc  )r;   r  s    r*   r   z.test_non_recursive_df_reduce.<locals>.<lambda>  s    Jquuw{{}5 r,   Fzcommit-dataset)chunk	aggregatesplit_everytokenr*  )rV   rW   r^   r  Seriesrp   rq   	reductionobjectr   r  )r7   r8   r9   r:   rd   re   Nrd  dask_seriesrK   r  s             @r*   test_non_recursive_df_reducer  n  s      
		X	&B			-	.B	78  	AYYQC!G5AE?Y;F..R.8K""5 # F !))F##((C///#s   CC 	C
C c                r   t        j                  d       t        j                  d       	 t        j                  j	                  ddd      j                         j                  dt        i      }|j                  dd	      }|j                          y # t        $ r t        j                  d
       Y y w xY w)NrP   rQ   
2000-01-01
2000-07-0112hr   r  r  	timestampT)sortedz(dd.set_index triggered a recursion error)rV   rW   r_   r  r  reset_indexrl  rO  r   r   RecursionErrorfail)r7   rh   s     r*   !test_set_index_no_resursion_errorr    s    
!
()	@MM$$<\PU$V[]V[#&' 	
 mmKm5 @>?@s   A(B B65B6c                     d} t        j                  t        |       5  t        d       d d d        t        j                  t        |       5  t        d       d d d        y # 1 sw Y   :xY w# 1 sw Y   y xY w)Nz	no Clientr  r   rn   r   )rV   rb   r  r   )msgs    r*   -test_get_scheduler_without_distributed_raisesr    se    
C	|3	/ 4 234 
|3	/ /./ /4 4/ /s   A$A0$A-0A9c                    t               | j                  k(  sJ d}t        j                  t        |      5 }t        d       t        d       d d d        t              dk(  sJ y # 1 sw Y   xY w)NzkRunning on a single-machine scheduler when a distributed client is active might lead to unexpected results.r  threadsrn   r   rC   )r   getrV   rQ  UserWarningr  )r7   warning_messageuser_warnings_as      r*   *test_get_scheduler_with_distributed_activer     sl    ?aee###	6  
k	9 (_	*'( 1$$$( (s   A))A2c                   t               | j                  k(  sJ t        j                  j	                  d      5  t        j                  t              5  t               | j                  k7  sJ 	 d d d        t        j                  j	                  d       5  t               | j                  k(  sJ 	 d d d        d d d        y # 1 sw Y   UxY w# 1 sw Y   xY w# 1 sw Y   y xY w)Nr  rn   )r   r  r_   r   r   rV   rQ  r  )r7   s    r*   7test_get_scheduler_with_distributed_active_reset_configr"    s    ?aee###	9	- ,\\+& 	, ?aee+++	,[[__t_, 	, ?aee+++	,, ,	, 	,	, 	,, ,s;   CC/(CC2CC	CC	CC$zscheduler, expected_classes))N)SerializableLockr#  AcquirerProxy)r  )r#  r#  r#  )r  )r$  r$  r$  c                   t        j                  d       t        j                  d       t        j                  dd      }t        j                  dd      }t        j                  dd	      }|j                  d
      }|j                  |dg      } |j                  dd      }t        |||f|      D ]-  \  }}	t        ||       }
|
j                  j                  |	k(  r-J  y )Nr   rP   r   Requires dask.arrayr%   zdask.bagzRequires dask.bagrQ   Requires dask.dataframed   r;   r  r)  rC   rl   rn   )	rV   rW   r  from_dask_arrayrp   zipr   r   r   )ro   expected_classesr   r   re   r8  rh   dbag
collectionr  rM   s              r*   test_get_scheduler_lockr0    s      
!			\2G	HB			Z0C	DB			-6O	PB776?D


TC5

1C288CQ'D #S$$57G H 2
H yA}}%%1112r,   multiprocessing_method)spawnfork
forkserverc                   t        j                  d       t        j                  d       t        j                  dd      }t        j                  dd      }|j                  d      }|j                  |d	g
      }t	        j
                  dd      }t        j                  j                  d|i      5  |||fD ]5  }t        |d      }t        |t        j                  j                        r5J  	 d d d        y # 1 sw Y   y xY w)Nr   rP   r   r&  r%   rQ   r'  r(  r;   r*  r)  rC   rl   z)distributed.worker.multiprocessing-methodr   rn   )rV   rW   r  r+  r   rp   r_   r   r   r   rJ   r   lockLock)	r7   r1  r   re   r8  rh   r.  r/  rM   s	            r*   #test_get_scheduler_lock_distributedr8    s      
!			\2G	HB			-6O	PB776?D


TC5

1C88CQ'D		46LM
 : d+ 	:J$Z=ICc;#3#3#8#8999	:: : :s   6:C>1C>>D
lock_paramc                B   t        j                  d       t        j                  d       t        j                  d       t        d      5 }t        j                  j                  ddd	      }|j                  t        |      d
|       ddd       y# 1 sw Y   yxY w)z_https://github.com/dask/dask/issues/9972 and
    https://github.com/dask/dask/issues/10315
    rP   rQ   rI  hd5)	extensionr  r  r  r  z/ds_*)r5   r6  N)rV   rW   r!   r_   r  r  rX  rO  )r7   r9  r   rh   s       r*   test_write_single_hdfr=    s     !
()
!	5	! 9Qmm&&\|RW&X

3q6wZ
89 9 9s   A BBro   r   )r   r  c                  K   t        j                  t              5  t        j                  j                         t        j                  j                  k(  sJ t        j                  j                  d      5  t        j                  j                         t        j                  j                  k(  sJ t        | j                  dd       d {   }	 t        j                  j                         t        j                  j                  k(  sJ 	 |j                          d {    t        | j                  dd       d {   }	 t        j                  j                         |j                  k(  sJ 	 |j                          d {    t        j                  j                         t        j                  j                  k(  sJ t        | j                  dd       d {   }	 t        j                  j                         t        j                  j                  k(  sJ |j                         5  t        j                  j                         }||j                  k(  sJ 	 d d d        t        j                  j                         t        j                  j                  k(  sJ 	 |j                          d {    t        | j                  dd       d {   }t        | j                  dd       d {   }	 |j                         5  t        j                  j                         |j                  k(  sJ 	 d d d        t        j                  j                         |j                  k(  sJ 	 |j                          d {    |j                          d {    t        j                  j                         t        j                  j                  k(  sJ 	 d d d        t        j                  j                         t        j                  j                  k(  sJ t        | j                  dd       d {   }	 t        j                  j                         |j                  k(  sJ t        j                  j                  d      5  t        j                  j                         t        j                  j                  k(  sJ |j                         5  t        j                  j                         |j                  k(  sJ 	 d d d        d d d        |j                          d {    	 d d d        y 7 u7 ## |j                          d {  7   w xY w7 $7 # |j                          d {  7   w xY w7 # 1 sw Y   )xY w7 # |j                          d {  7   w xY w7 7 # 1 sw Y   xY w7 C7 .# |j                          d {  7   |j                          d {  7   w xY w# 1 sw Y   )xY w7 # 1 sw Y   xY w# 1 sw Y   xY w7 # |j                          d {  7   w xY w# 1 sw Y   y xY ww)Nr  rn   FT)set_as_defaultr  )rV   rQ  r  r_   baser   localget_syncr   r   threadedr  r   addressclose
as_current)r8   r   scclient2s       r*   5test_get_scheduler_default_client_config_interleavingrI    s     
k	" 7!yy&&(DJJ,?,????[[__y_1 )	B99**,0A0AAAA!!))EPTUUF%yy..0DMM4E4EEEElln$$!!))DtTTF%yy..0FJJ>>>lln$$99**,0A0AAAA "!))EPTUUF%yy..0DMM4E4EEEE&&( ,002B+++, yy..0DMM4E4EEEElln$$ "!))DtTTF"199UQUVVG&'') D99224CCCDyy..0FJJ>>>lln$$mmo%%99**,0A0AAAAS)	BV yy&&(DJJ,?,????aii4PP	!99**,

:::95 Cyy..0DMM4E4EEEE&&( C99224

BBBCC
 ,,.  o7! 7! V %flln$$T %flln$$ V, ,
 %flln$$ UVD D %% lln$$mmo%%O)	B )	BZ Q
C CC C
 !&,,.  o7! 7!s  Y$AY8AXUX;U
X$U%XU(X
-U.8XU+AX&V'X,AV70V(AV,X?V XV= X?W  XW.W5W:XWX%W&?X&AYXY
AX:AX*#.XX*X:"Y5X76Y;	Y$XX
U%U!U%%X+X.V	VV		XVVXV:3V64V::X XWWXXX
*W-+X
XX

XX		YX'"X**X4	/X:7Y:YY
YYY!Y$c                   K   t        j                  dd      }|j                  d       }t        d |j                  D              rJ y w)Nr)  r4   rl   c                    | dz  S )N   r   r%  s    r*   r   z*test_bag_groupby_default.<locals>.<lambda>7  s
    QV r,   c              3  *   K   | ]  }d |d   v   yw)partdr   Nr   )r  ks     r*   r  z+test_bag_groupby_default.<locals>.<genexpr>8  s     4q7ad?4s   )r   rp   r   r  r_   )r7   r8   r9   r:   b2s        r*   test_bag_groupby_defaultrQ  4  s@     
"%A	
#	$B4BGG44444s   A	A)s
__future__r   rV   rW   r   r   rU  sys	functoolsr   operatorr   r   r   r	   distributed.utils_testr
   r   r7   r   r   r   r   r   r   r   r_   dask.bagbagr   r   r   r   	dask.baser   r   dask.blockwiser   dask.delayedr   r)   r   r   dask.layersr   r   
dask.utilsr   r   r    r!   dask.utils_testr"   markskipifplatform
pytestmarkfilterwarningsignore_sync_scheduler_warningr+   r1   r@   rN   ri   rs   parametrizer   r   r   r   r   r   xfail_with_pyarrow_stringsr   paramrL  r   r   r   r   r  r	  r  r  r  r  r1  r;  skip_with_pyarrow_stringsflakyra  rf  rx  r|  r  r  r  r  r  r  r  r  r  r  r  r  r   r"  r0  r8  r6  r7  r=  rI  rQ  r   r,   r*   <module>rj     si   " !f!!-0  	 
   = = * .     * * = $   - 8 J J >+66+%@Kg%8G [[LLG	(   
 !' : :2! (0 D0 0&&$	- $/ 04 cU|,tUm4 5 -&   )O )OX; D   '' ( 	++##+GPU#V	
 	++##+GPU#V	

 ''. (  . D "	S
%%,&
R D0 0 D$ $>  !45. 6.6 !)N)N	
 	U&++"3"31"3"=> !45$7,5 8 6 ,5^.& D " D /, D   F$2 D	2 	2 DW W, %	 	
L DG9q=1%8 2%8P D> >()X 2 D
 	

. D0 02@ /	%, !22  ::$ &&k.>.>.C.C.E'FG	9 H '	9 [&)B7:! 8:!z D5 5r,   