
    tfށ              
      
   d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlm	Z	 d dl
mZ d dlmZmZ d dlmZ d dlZd dlmZmZ d dlZd dlmZ d dlmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z) d d	l*m+Z+m,Z, d d
l-m.Z. d dl/m0Z0 d dl1m2Z2m3Z3 d dl4m5Z5m6Z6m7Z7  e6d      Z8 e6d      Z9 e6d      Z: e6d      Z;dZ<dZ=d Z>d Z?d Z@d ZAd ZBd ZCd ZD G d de      ZEd ZFd ZGej                  j                  d      d         ZJej                  j                  d      d!        ZKej                  j                  d      d"        ZLej                  j                  d#      d$        ZMej                  j                  d#      d%        ZNej                  j                  d#      d&        ZOej                  j                  d#      d'        ZPej                  j                  d#      d(        ZQej                  j                  d#      d)        ZRej                  j                  d#      d*        ZSej                  j                  d+      d,        ZTej                  j                  d#      d-        ZUej                  j                  d#      d.        ZVej                  j                  d      d/        ZWej                  j                  d      d0        ZXd1 ZYej                  j                  d      ej                  j                   eZej                  jH                        d23      ej                  j                  ej                  d4k(  d5d67      d8                      Z^ej                  j                  d      ej                  j                   eZej                  jH                        d23      d9               Z_ej                  j                  d      ej                  j                   eZej                  jH                        d23      d:               Z`d; Zad< Zbd= Zcd> Zdd? Zed@ ZfdA ZgdB Zhej                  j                  dCdDdEe<fg      dF        Zjej                  j                  dGdDi dDfdDdHdIidDfdDdDdJidJfdEe<fdEdKidKe<ffg      dL        ZkdM ZldN Zmej                  j                  d      dO        ZndP ZodQ ZpdR ZqdS ZrdT ZsdU ZtdV ZudW Zv G dX dYe      ZwdZ Zxd[ Zyej                  j                  d\d]^      ej                  j                  ej                  j                  d_d`dag      db                      Z|dc Z}dd Z~de Zdf Zej                  j                  dgdh      di        Zy)j    )annotationsNOrderedDict)Executor)addmul)
NamedTuple)mergepartial)DaskMethodsMixin	clone_keycollections_to_dskcomputecompute_as_if_collectionget_collection_namesget_name_from_keyget_scheduleris_dask_collectionnamed_schedulersoptimizepersistreplace_name_in_keytokenizeunpack_collections	visualize)Delayeddelayed)Profiler)HighLevelGraph)tmpdirtmpfile)decimport_or_noneinc
dask.arraydask.dataframenumpypandas)g333333?foo)   h2c                      G d d      } t        d      dz   }t        |      sJ t        d      rJ t         | i             sJ t        |       rJ y )Nc                      e Zd Zd Zd Zy)0test_is_dask_collection.<locals>.DummyCollectionc                    || _         y Ndask)selfdsks     ^/var/www/html/software/conda/envs/higlass/lib/python3.12/site-packages/dask/tests/test_base.py__init__z9test_is_dask_collection.<locals>.DummyCollection.__init__7   s	    DI    c                    | j                   S r0   r1   r3   s    r5   __dask_graph__z?test_is_dask_collection.<locals>.DummyCollection.__dask_graph__:       99r7   N)__name__
__module____qualname__r6   r:    r7   r5   DummyCollectionr.   6   s    		r7   r@         )r   r   )r@   xs     r5   test_is_dask_collectionrD   5   s[      	
QAa   !!$$$ob1222!/2222r7   c                     t        j                  d      } t        j                  d      }| j                  g d      }|j                  |      }t	        |      rJ t	        |      sJ y )Nr(   	dask_exprrA   rB   r*   )pytestimportorskipSeriesfrom_pandasr   )pddxdfdxfs       r5   !test_is_dask_collection_dask_exprrP   D   sZ    			X	&B			[	)B	9	B
..
C!"%%%c"""r7   c                    t        j                  d       t        j                  d      }  G d d| j                  j                        }| j                  j                   |             }t        j                  t        d      5  |j                          d d d        t        j                  t        d      5  |j                          d d d        t        |      sJ y # 1 sw Y   KxY w# 1 sw Y   #xY w)Nr(   rF   c                  0    e Zd Zed        Zd Zd ZddZy)Ptest_is_dask_collection_dask_expr_does_not_materialize.<locals>.DoNotMaterializec                     y)Nr   r?   r9   s    r5   _metazVtest_is_dask_collection_dask_expr_does_not_materialize.<locals>.DoNotMaterialize._metaS   s    r7   c                    J d       Nmust not reachr?   r9   s    r5   __dask_keys__z^test_is_dask_collection_dask_expr_does_not_materialize.<locals>.DoNotMaterialize.__dask_keys__W       ***5r7   c                    J d       rW   r?   r9   s    r5   r:   z_test_is_dask_collection_dask_expr_does_not_materialize.<locals>.DoNotMaterialize.__dask_graph__Z   rZ   r7   c                    J d       rW   r?   )r3   fuses     r5   r   zYtest_is_dask_collection_dask_expr_does_not_materialize.<locals>.DoNotMaterialize.optimize]   rZ   r7   N)F)r<   r=   r>   propertyrU   rY   r:   r   r?   r7   r5   DoNotMaterializerS   R   s#    		 
		+	+	+r7   r_   rX   match)rH   rI   _coreExpr_collectionnew_collectionraisesAssertionErrorrY   r:   r   )rM   r_   colls      r5   6test_is_dask_collection_dask_expr_does_not_materializeri   N   s    
!			[	)B+288== + >>(()9);<D	~-=	> 	~-=	>  d###  s   C 9C, C),C5c            	     :  
 t         j                   G d d             
 G d dt              t        d      dz   } | dz   }| dz   }
fd} || ||d	 | ||fD              }t	        | \  }}t        |      d
k(  sJ  |g d      } |dddg d      }||k(  sJ t	        |ddi\  }}t        |      dk(  sJ  ||      |k(  sJ t	        dddd
i      \  }}|rJ  ||      dddd
ifk(  sJ d }	t	        | |	df|	dd
fgd      \  }} ||        ||	dfg       y )Nc                      e Zd ZU ded<   y)+test_unpack_collections.<locals>.ADataClassintaNr<   r=   r>   __annotations__r?   r7   r5   
ADataClassrl   k   s    r7   rq   c                      e Zd ZU ded<   y),test_unpack_collections.<locals>.ANamedTuplerm   rn   Nro   r?   r7   r5   ANamedTuplers   o   s    r7   rt   rA      rB   c                    | |d| | |ddd|ggddd|dfd| dd	hd
t        d| fg      d |       d| fd |       i
|f}|S )Nrn   brA   rB   c
   der*   fg)rn   hir   )rn   rw   rx   iteratortrq   rt   s        r5   buildz&test_unpack_collections.<locals>.buildv   s}    Q1aQC[RaVaAY[3(,Z!_j!_[1% 
$ r7   c              3      K   | ]  }|  y wr0   r?   ).0r   s     r5   	<genexpr>z*test_unpack_collections.<locals>.<genexpr>   s     010s   r*   )~a~b~cr   r   r   traverseFrn   c                     t        d      )NzShouldn't have been called)
ValueError)argss    r5   failz%test_unpack_collections.<locals>.fail   s    566r7   r   )dataclasses	dataclassr	   r   r   len)rn   rw   rx   r   r   collectionsrepackresultsolr   rq   rt   s             @@r5   test_unpack_collectionsr   j   sz     j  	
QA	AA	AA* Aq0q!Qi01D,d3K{q    &'F
dD"4
5CS== -dCUCK{q   +$&&& -QC8<K?+1a#q"22227 -	D!9a|nuK ;
T1I;r7   c                     G d d      } t        j                  t              5  t        t	                      d d d        t        j                  t              5  t         | ddidg             d d d        t        j                  t              5  t         | ddidg             d d d        t        j                  t              5  t         | ddidg             d d d        t         | i g             t               k(  sJ t         | dt        fddt        fddd	d
didt        fdt        fdd
ggg            h dk(  sJ y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w)Nc                      e Zd Zd Zd Zd Zy)2test_get_collection_names.<locals>.DummyCollectionc                     || _         || _        y r0   )r2   keysr3   r4   r   s      r5   r6   z;test_get_collection_names.<locals>.DummyCollection.__init__   s    DIDIr7   c                    | j                   S r0   r1   r9   s    r5   r:   zAtest_get_collection_names.<locals>.DummyCollection.__dask_graph__   r;   r7   c                    | j                   S r0   )r   r9   s    r5   rY   z@test_get_collection_names.<locals>.DummyCollection.__dask_keys__   r;   r7   N)r<   r=   r>   r6   r:   rY   r?   r7   r5   r@   r      s    			r7   r@   rA   rB   r?   rA   a-1b-2r*   rx      >   rx   r   r   )rH   rf   	TypeErrorr   objectseth1r+   )r@   s    r5   test_get_collection_namesr      s[   	 	 
y	! 'VX&' 
y	! ;_aVaS9:;	y	! =_b!Wrd;<=	y	! A_dAY?@A  B 78CEAAA  R[!eR[!UAsA>r{UBK456	

 
  ' '; ;= =A As/   EEEE)EEE&)E2c                    t        d      dk(  sJ t        d      sJ d       t        dt        t        f      dk(  sJ t        j                  t
              5  t        d       d d d        t        j                  t
              5  t        d       d d d        t        j                  t
              5  t        d       d d d        y # 1 sw Y   dxY w# 1 sw Y   CxY w# 1 sw Y   y xY w)Nr)   foo-123rA   r?   r   )r   r   r+   rH   rf   r   r?   r7   r5   test_get_name_from_keyr      s    U#u,,,Y'22'iR01Y>>>	y	! !	y	! "	y	!  $   	     s$   CC1CCCC'c                 H   t        di       dk(  sJ t        dddi      dk(  sJ t        dddd      dk(  sJ t        dddi      dk(  sJ t        dt        t        fddi      dt        t        fk(  sJ t        j                  t
              5  t        di        d d d        t        j                  t
              5  t        d	i        d d d        t        j                  t
              5  t        d
i        d d d        y # 1 sw Y   fxY w# 1 sw Y   DxY w# 1 sw Y   y xY w)Nr)   barbazasd)r)   r   r   zbar-456rA   r?   r   )r   r   r+   rH   rf   r   r?   r7   r5   test_replace_name_in_keysr      s    ub)U222uuen5>>>ueE&BCuLLLy9i*@AYNNN	2r2Y4FG

L   
 
y	! #Ar"#	y	! $B#$	y	! &D"%& &	# #$ $& &s$   D <D*D D	DD!c                      e Zd ZdZ eej                  j                        Zd Z	d Z
d Zd Zd Zd Zd Zd	 Zed
dd       Zy
)Tuple_dask_keysc                     || _         || _        y r0   r   r   s      r5   r6   zTuple.__init__   s    

r7   c                    t        |t              st        S t        t        | j                  |j                        | j
                  |j
                  z         S r0   )
isinstancer   NotImplementedr
   r   r   )r3   others     r5   __add__zTuple.__add__   s;    %'!!U4::u{{3TZZ%++5MNNr7   c                    | j                   S r0   )r   r9   s    r5   r:   zTuple.__dask_graph__       zzr7   c                    | j                   S r0   r   r9   s    r5   rY   zTuple.__dask_keys__   r   r7   c                *    t        t        |             S r0   )tupler   r9   s    r5   __dask_layers__zTuple.__dask_layers__   s    )$/00r7   c                    | j                   S r0   r   r9   s    r5   __dask_tokenize__zTuple.__dask_tokenize__   r   r7   c                    t         dfS )Nr?   )r   r9   s    r5   __dask_postcompute__zTuple.__dask_postcompute__  s    byr7   c                <    t         j                  | j                  ffS r0   )r   _rebuildr   r9   s    r5   __dask_postpersist__zTuple.__dask_postpersist__  s    ~~

},,r7   N)renamec               Z    |r|D cg c]  }t        ||       }}t        | |      S c c}w r0   )r   r   )r4   r   r   keys       r5   r   zTuple._rebuild  s3    @DE'V4EDES$ Fs   ()r<   r=   r>   	__slots__staticmethodr2   threadedget__dask_scheduler__r6   r   r:   rY   r   r   r   r   r   r?   r7   r5   r   r      s\    "I%dmm&7&78O
1- &*    r7   r   c                 H	   dt         fddt        fdi} dt         ft        dt         fdt        ffdt        ft        dt         fdfi}|j                  |        dt        dt         fdt        ffi}|j                  |       t	        i g       }t	        | dt         fdt        fg      }t	        |dt         fdt        fg      }t	        |dg      }||z   |z   |z   }t        j                  t              5  d|_        d d d        t        |      sJ t        |      sJ t        |      sJ t        |      sJ t        |      sJ t        |      t        |      k(  sJ t        |      t        |      k(  sJ t        |      t        |      k(  sJ t        |      t        |      k(  sJ t        |      t        |      k(  sJ t        |||||fD ch c]  }t        |       c}      dk(  sJ t        |      t               k(  sJ t        |      dhk(  sJ t        |      dhk(  sJ t        |      dhk(  sJ t        |      h dk(  sJ |j                         dk(  sJ |j                         d	k(  sJ |j                         d
k(  sJ |j                         dk(  sJ t        j                  |d|i||g      ddd	id
dgfk(  sJ |j                         dk(  sJ |j!                         }	t#        |	t              sJ |	j$                  |j$                  k(  sJ t'        |	j(                  j+                               g dk(  sJ |	j                         dk(  sJ t        j                   ||||      \  }
}}}|j$                  |j$                  k(  sJ |j(                  dt         fddt        fdik(  sJ |j                         d
k(  sJ ||z   |z   }|j                         dk(  sJ |j-                         \  }} |i g|dddii}|j                         dk(  sJ |j-                         \  }} |dt         fddt        fdig|dddii}|j                         dk(  sJ |j-                         \  }} |ddig|dddii}|j                         dk(  sJ y # 1 sw Y   xY wc c}w )NrC   rA   rB   yzru   >   rC   r   r   r?   )rA   rB   )r*   r   )   )rA   rB   r*   r   r   r*   r   r   ww3x3ry      )ry   r   z3F   )r   )r   r+   r   updater   rH   rf   AttributeErrorr)   r   r   r   r   r   r   r2   r   r   r   sortedr   valuesr   )r4   dsk2dsk3r   rC   r   r   r   rh   t2w2x2y2z2t3rebuildr   r   r   r   s                       r5   test_custom_collectionr     s   9a#rA
&C"Ic2Yb	2S"Ic2YPQ?RSDKK#Ry3),-DKKb"AcS"IRy)*Adc2Yb	*+AdSEA	A	AA 
~	&  a   a   a   a   a    A;(1+%%%A;(1+%%%A;(1+%%%A;(1+%%%A;(1+%%%Aq!Q?;4;<AAA  "ce+++"se+++"se+++"se+++"o555 99;"99;&   99;&   99;$<<S!Ha+,sFmVT5R0SSSS99;/))) 
Bb%   88qww"((//#$777::<?***\\!Q1-NBB88qww88b	1sBi3333::<6!!!	b2B::<?*** **,MGT		/d	/C;	/B::<2**,MGT	4*b4*b1	MD	M#t	MB::<8###**,MGT	$	7d	7C;	7B::<5   y " <s   /RRRc                 V   ddl m}  t        j                  t	        d      d      }t        t        d      }t        t        d      }|j                  |      j                  |      }g  | fd	      5  |j                  d
d       d d d        t        D cg c]  }d|d   v s| c}      dk(  sJ t        D cg c]  }d|d   v s| c}      dk(  sJ g  | fd	      5  |j                  d
       d d d        t        D cg c]  }d|d   v s| c}      dk(  sJ t        D cg c]  }d|d   v s| c}      dk(  sJ t        D cg c]  }d|d   v s| c}      dk(  sJ y # 1 sw Y   xY wc c}w c c}w # 1 sw Y   xY wc c}w c c}w c c}w )Nr   )Callbackd   r   npartitionsrA   rB   c                &    j                  |       S r0   appendr   r   r   s     r5   <lambda>z%test_compute_no_opt.<locals>.<lambda>i      T[[-= r7   )pretasksingle-threadedF)	scheduleroptimize_graphr   r   c                &    j                  |       S r0   r   r   s     r5   r   z%test_compute_no_opt.<locals>.<lambda>o  r   r7   r      zadd-mul)dask.callbacksr   dbfrom_sequenceranger   r   r   mapr   r   )r   rw   add1mul2okr   s         @r5   test_compute_no_optr  ]  s    (
s3A3?D3?D	dAD	=	> E			-e	DE41a5AaD=12a77741a5AaD=12a777D	=	> /			-	./ 41a5AaD=12a77741a5AaD=12a77745a9!#456!;;;E E11/ / 215sT   6E:F(FFF,FFF6F!F!F&)F&:FFznot dac                    t         j                  d      j                  d      } t        j	                  | d      }|dz   }|dz   }t        ||      \  }}t         j                  || dz         sJ t         j                  || dz         sJ y )Nr   ry   ry   ru   ru   chunksrA   rB   )nparangereshapeda
from_arrayr   allclose)arrdarrdarr1darr2out1out2s         r5   test_compute_arrayr  x  s~    
))C.
 
 
*C==V=,D1HE1HE&JD$;;tS1W%%%;;tS1W%%%r7   c                    ddl m}  t        j                  d      j	                  d      }t
        j                  |d      }|dz   |j                  d      z
  }|j                         } | ||       t        |j                        j                  |j                        sJ t        |j                        |j                  k(  sJ y )	Nr   	assert_eqr   r  r  r  rA   )axis)dask.array.utilsr  r
  r  r  r  r  meanr   r   r2   issubsetr   r   )r  r  rC   r   s       r5   test_persist_arrayr    s    *
))C.
 
 
*C
c&)A	
Q!&&a&. A			AaOqvv;'''qvv;!--'''r7   c                    t         j                  dt        d      } | j                         \  }}t        j                  ddg      t        j                  ddg      d} ||g|d| j                  dii}t        |t         j                        sJ |j                  dk(  sJ |j                         d	d
gk(  sJ t         j                  j                  |g d       y )Nr   rB   )dtyper	  rA   r*   rw   r   rw   rA   r   rw   r"  r#  rA   rB   r*   r   )r  zerosrm   r   r
  arraynamer   ArrayrY   utilsr  rn   r   r   r4   rw   s        r5   test_persist_array_renamer+    s    
#a(A**,MGTXXq!f%1a&1A
BC1d1AFFC=1Aa"""66S==??8 4444HHq,'r7   znot ddc                    t         j                  g dg dd      } t        j                  | d      }|j                  dz   }|j                  |j
                  z   }t        ||      \  }}t        j                  j                  || j                  dz          t        j                  j                  || j                  | j
                  z          y )Nr$  ru   ru   r*   r*   rn   rw   rB   r   rA   )	rL   	DataFrameddrK   rn   rw   r   r)  r  )rN   ddfddf1ddf2r  r  s         r5   test_compute_dataframer4    s    	L|<	=B
...
+C5519D55355=Dt$JD$HHtRTTAX&HHtRTTBDD[)r7   c                    t         j                  g dg dd      } t        j                  | d      dz  }t	        |j                               dk(  sJ |j                         }t        |t        j                        sJ t        j                         st	        |j                               dk(  sJ 	 t        j                  j                  ||       y )Nr$  ru      r   r   r.  rB   r   r   )rL   r/  r0  rK   r   r:   r   r   _dask_expr_enabledr)  r  )rN   r2  r3  s      r5   test_persist_dataframer9    s    	L|<	=B>>"!>,q0Dt""$%***<<>DdBLL)))242G2G2I3t""$%*PPqHHtT"r7   c                    t         j                  g d      } t        j                  | d      dz  }t	        |j                               dk(  sJ |j                         }t        |t        j                        sJ t        j                         st	        |j                               dk(  sJ 	 t        j                  j                  ||       y )Nr$  rB   r   r   )rL   rJ   r0  rK   r   r:   r   r   r8  r)  r  )dsdds1dds2s      r5   test_persist_seriesr>    s    	<	 B>>"!>,q0Dt""$%***<<>DdBII&&&242G2G2I3t""$%*PPqHHtT"r7   c                    dd l m}  t        j                  g d      }| j	                  |d      j                         }| j                         st        |j                               dk(  sJ 	 |j                         }| j                         s"t        || j                  j                        sJ | j                         st        |j                               dk(  sJ 	 | j                  j                  ||       y )Nr   r$  rB   r   ru   rA   )dask.dataframe	dataframerL   rJ   rK   minr8  r   r:   r   r   coreScalarr)  r  )r0  r;  r<  r=  s       r5   test_persist_scalarrE    s    	<	 B>>"!>,002D242G2G2I3t""$%*PPq<<>D  "$///242G2G2I3t""$%*PPqHHtT"r7   c                    dd l m}  | j                         rt        j                  d       t
        j                  g dg dd      }t
        j                  g dg dd      }| j                  |d	      }|j                         \  }}|j                  d d |j                  dd  d
} ||g|d|j                  dii}|j                         ddgk(  sJ | j                  j                  ||       y )Nr   doesn't make senser$  r6  r.  )rB   r*   ru   r7  )r7  r   	   ry   rB   r   rC   r   rC   rA   r   rC   rJ  rK  )r@  rA  r8  rH   skiprL   r/  rK   r   iloc_namerY   r)  r  )r0  df1df2r2  r   r   r4   r3  s           r5   test_persist_dataframe_renamerQ    s    	()
,,\=
>C
,,\>
?C>>#1>-D--/MGTXXbq\SXXab\
:C388tzz3&78DHh#7777HHtS!r7   c                    dd l m}  | j                         rt        j                  d       t
        j                  g d      }t
        j                  g d      }| j                  |d      }|j                         \  }}|j                  d d |j                  dd  d} ||g|d|j                  d	ii}|j                         d
dgk(  sJ | j                  j                  ||       y )Nr   rG  r$  r6  rB   r   rI  r   rC   rJ  rK  )r@  rA  r8  rH   rL  rL   rJ   rK   r   rM  rN  rY   r)  r  )r0  ds1ds2r<  r   r   r4   r=  s           r5   test_persist_series_renamerU    s    	()
))L
!C
))L
!C>>#1>-D--/MGTXXbq\SXXab\
:C388tzz3&78DHh#7777HHtS!r7   c                    dd l m}  | j                         rt        j                  d       t
        j                  g d      }| j                  |d      j                         }|j                         \  }} |ddig|d|j                  d	ii}|j                         dgk(  sJ | j                  j                  |d       y )
Nr   rG  r$  rB   r   rJ  ru   r   rC   )r@  rA  r8  rH   rL  rL   rJ   rK   rB  r   rN  rY   r)  r  )r0  rS  r<  r   r   r=  s         r5   test_persist_scalar_renamerW    s    	()
))L
!C>>#1>-113D--/MGTHa=B4BS0ABDH:---HHtQr7   znot dd or not dac                    t         j                  d      j                  d      } t        j	                  | d      dz   }t
        j                  g dg dd      }t        j                  |d	
      j                  d	z   }t        ||      \  }}t         j                  || dz         sJ t        j                  j                  ||j                  d	z          y )Nr   r  r  r  rA   r$  r-  r.  rB   r   )r
  r  r  r  r  rL   r/  r0  rK   rn   r   r  r)  r  )r  r  rN   r1  arr_outdf_outs         r5   test_compute_array_dataframer[    s    
))C.
 
 
*C==V=,q0D	L|<	=B
...
+
-
-
1CdC(OGV;;wa(((HHvrttax(r7   c                     t         j                  t        j                  j                  d      dj	                         g      } t
        j                  | d       y )Nr*   rA      ödatacolumnsr   r   )rL   r/  r
  randomencoder0  rK   rN   s    r5   -test_compute_dataframe_valid_unicode_in_bytesre    s;    	299++F3dkkm_	MBNN21N%r7   c                     t         j                  t        j                  j                  d      dg      } t        j                  | d       y )Nr]  u   r_  r   r   )rL   r/  r
  rb  r0  rK   rd  s    r5   &test_compute_dataframe_invalid_unicoderg    s7     
299++F3hZ	HBNN21N%r7   c                 *   t         j                  dd      t        j                  g d      t	        j
                  t        fd       t        d      \  } }t        j                  | t        j                  d            sJ |g dk(  sJ y )Nru   rB   r  rG   c                     t               S r0   )r   )rw   rC   s   r5   r   z(test_compute_array_bag.<locals>.<lambda>  s    gam r7   r   r   )
r  r  r   r   rH   rf   r   r   r
  r  )xxbbrw   rC   s     @@r5   test_compute_array_bagrl    so    
		!A	A
#A
MM*34Q%67FB;;r299Q<(((??r7   c                     t         j                  dd      } d}t        | |      \  }}|| j                         k(  j                         sJ ||k(  sJ t        d      dk(  sJ y )Nru   rB   r  ry   ru   )r  r  r   all)rC   r   rj  yys       r5   test_compute_with_literalrq  !  sb    
		!A	A
AQ]FB!))+""$$$7N71:r7   c                     t        d      dz   } | dz   }| dz   }t        | dd|gd|df      dg dddfk(  sJ t        | |g|d	      }|d
   d
   | u sJ |d
   d   |u sJ |d   dk(  sJ y )NrA   ru   rB   r.  r7  rA   rB   r   r   rB   Fr   r   r   r   r   )rn   rw   rx   ress       r5   test_compute_nestedrw  -  s    
QA	AA	AA!Q+aV4i 9   
 1a&!e
,Cq6!9>>q6!9>>q6Q;;r7   z'graphviz exception with Python -OO flag)reasonwin32z:graphviz/pango on conda-forge currently broken for windowsF)rx  strictc            
        t        j                  d       t        j                  d       t               5 } t        j	                  dd      }|j                  t        j                  j                  | d             t        j                  j                  t        j                  j                  | d            sJ |j                  t        j                  j                  | d	             t        j                  j                  t        j                  j                  | d	            sJ t        |d
dt        j                  j                  | d             t        j                  j                  t        j                  j                  | d            sJ d
t        ddft        dd
fd}t        ||t        j                  j                  | d             t        j                  j                  t        j                  j                  | d            sJ t        |g d      }t        |t        j                  j                  | d             t        j                  j                  t        j                  j                  | d            sJ t        |g d      }t        |t        j                  j                  | d      d       t        j                  j                  t        j                  j                  | d            sJ t        |t        j                  j                  | d      d       t        j                  j                  t        j                  j                  | d            sJ t        j                  j                  d      5  t        |t        j                  j                  | d             t        j                  j                  t        j                  j                  | d            sJ 	 d d d        t        j                   t"        d      5  t        |d       d d d        |j                  d        d d d        y # 1 sw Y   UxY w# 1 sw Y   0xY w# 1 sw Y   y xY w)Ngraphvizipycytoscaperu   rB   r  mydaskfilename
mydask.pngz
mydask.pdfrA   rn   )rn   rw   rx   cyt	cytoscape)r  enginezcyt.htmlz	cyt2.html)visualization__enginez	cyt3.htmlznot-realr`   )r  )rH   rI   r    r  r  r   ospathjoinexistsr   r   r   r2   configr   rf   r   )rz   rC   r4   s      r5   test_visualizer  <  s    
#
'	 ##QIIaI"	RWW\\!X67ww~~bggll1l;<<<	RWW\\!\:;ww~~bggll1l;<<<!QBGGLLL$ABww~~bggll1l;<<<S#qMc1>!S277<<<#@Aww~~bggll1l;<<<#'!bggll1l;<ww~~bggll1l;<<<#'!bggll1e4[Iww~~bggll1j9:::!bggll1k:>Rww~~bggll1k:;;;[[__;_? 	@a"'',,q+">?77>>"'',,q+">???	@ ]]:Z8 	,a
+	,
 	
T"G## ##4	@ 	@	, 	,=## ##s=   MQ*A+Q>#Q*!Q/Q*Q	Q*Q'	#Q**Q3c                 @   t        j                  d      } t               5 }t        j	                  dd      }|j
                  j                  t        j                  j                  |d            }t        || j                        sJ 	 d d d        y # 1 sw Y   y xY w)Nr|  ru   rB   r  r  r  )rH   rI   r    r  r  r2   r   r  r  r  r   Digraph)r|  rz   rC   vizs       r5   test_visualize_highlevelgraphr  n  s~    
 "":.H	 1QIIaI"ffQ(EF#x//000	1 1 1s   A*BBc                 ^   t        j                  d       t        j                  d       t        j                  dd      } t	        d      5 }| j                  d|d	
       t        |      5 }|j                         }d d d        dv sJ 	 d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)Nr|  zmatplotlib.pyplotru   rB   r  dot)	extensionorderRdBu)colorr  cmapzcolor="#)rH   rI   r  r  r!   r   openread)rC   fnr|   texts       r5   test_visualize_orderr  {  s    
 
#
+,
		!A	A	5	! "R	'BV<"X 	668D	T!!!	" "	 	" "s$    B#.B?B#B 	B##B,c                p    t        |       } | D ]%  }| |   d   t        k(  st        f| |   dd  z   | |<   ' | S )Nr   rA   )dictr$   r"   )r4   r   r   s      r5   
inc_to_decr    sK    
s)C -s8A;#vC,CH- Jr7   c                 6    t        j                  t              d      } | j                         dk(  sJ t         j                  j                  t        g      5  | j                         dk(  sJ 	 d d d        | j                         dk(  sJ y # 1 sw Y   xY w)NrA   rB   optimizationsr   )r2   r   r$   r   r  r   r  rC   s    r5   test_optimizations_keywordr    s~    S!A99;!	
|	4  yy{a  99;!   s   BBc                 z    t        j                  t              d      }  t        j                  t              |       }| |z   }t        | ||d      \  }}}}|dk(  sJ t	        |j                         }t	        |j                         |k(  sJ t	        |j                         |k(  sJ t        j
                  |||      t        j
                  | ||      k(  sJ t        j
                  | ||t        g      }t        | ||t        g      \  }	}
}t        j
                  |	|
|      |k(  sJ t         j                  j                  t        g      5  t        | ||      \  }}}d d d        t        |	|
|gg      D ]2  \  }}t	        |j                         t	        |j                         k(  r2J  y # 1 sw Y   QxY w)NrA   r  )
r2   r   r$   r   r  r   r  r  r   zip)rC   r   r   r   r   r   constantr4   solsr   y3r   x4y4z4rn   rw   s                    r5   test_optimizer    s   S!AS!A	AA#Aq!Q/BBq== rww-C=C=C <<B#t||Aq!'<<<< <<1a
|<D!Q*>JBB<<B#t+++ 

|	4 'aA&
B'RRL2r2,/ ,1AFF|tAFF|+++,' 's   F11F:c                     t        j                  t              d      }  t        j                  t              |       }| |z   }t        | dd|gd|df      }|d   d   }|d   d   d   }|d   d   }t	        |t
              sJ t	        |t
              sJ t	        |t
              sJ t        |j                         t        |j                         cxk(  rt        |j                         k(  sJ  J t        | dg dddfk(  sJ t        | |g|d	
      }|d   d   | u sJ |d   d   |u sJ |d   j                         dk(  sJ y )NrA   rB   r.  r   rn   rw   rG   )ru   rB   Fr   ru   )r2   r   r$   r   r   r   r  r   )rn   rw   rx   r   a2b2c2rv  s           r5   test_optimize_nestedr    sM   S!AS!A	AAAQ1I.A7F	3B	3	B	1Bb'"""b'"""b'"""=DM:T"'']:::::Fai 8&AAAA
Aq61u
-Cq6!9>>q6!9>>q6>>q   r7   c                     d} t        j                  t        j                  d| g      }t	        t        |j                                     }d|v sJ g d}|D ]  }||vrJ  y)zI
    Startup time: `import dask` should not import too many modules.
    zYif 1:
        import dask
        import sys

        print(sorted(sys.modules))
        -cr2   )r%   r&   r'   r(   partds3fsdistributedN)
subprocesscheck_outputsys
executabler   evaldecode)codeoutmodules	blacklistmods        r5   test_default_importsr    so    D 
!
!3>>4">
?C$szz|$%GWI  "'!!!"r7   c                 (    t        ddd      dk(  sJ y )NrA   rB   r*   rG   )r   r?   r7   r5   test_persist_literalsr    s    1ay(((r7   c                    t        d      dz   } | dz   }| dz   }t        | dd|gd|df      }t        |d   d   t              sJ t        |d   d   d   t              sJ t        |d   d   t              sJ t	        | dg d	dd
fk(  sJ t        | |g|d      }|d   d   | u sJ |d   d   |u sJ |d   j	                         dk(  sJ y )NrA   ru   rB   r.  r   rn   rw   r7  rs  rt  Fr   r   r   r   r   r   r   )rn   rw   rx   r   rv  s        r5   test_persist_nestedr    s    
QA	AA	AA1Aq!9-1v6FfQing...fQinQ'111fQilG,,,Fai 8&AAAA
1a&!e
,Cq6!9>>q6!9>>q6>>q   r7   c                 \   t        d      }  t        t              |       } t        t              |      }t        |      \  }t        |t              sJ |j
                  |j
                  k(  sJ t        |j                        dk(  sJ |j                         |j                         k(  sJ y NrA   )	r   r$   r   r   r   r   r   r2   r   )x1r   r   rj  s       r5   test_persist_delayedr    s    	B	b	B	b	BBKERb'"""66RVVrww<1::<2::<'''r7   r   rn   za-123c                    t        | | dddi      }|j                         dk(  sJ |j                         }|j                         dk(  sJ |j                  | k(  sJ t	        |j
                        | dik(  sJ y )Nrw   rA   )r   r   r   r   r  r2   )r   rz   dps      r5   test_persist_delayed_custom_keyr    st    c3Q'(A99;!	
B::<166S===S!H$$$r7   zkey,rename,new_keyrx   rz   rw   zb-123c                   t        | | di      }|j                         dk(  sJ |j                         \  }} ||dig|d|i}|j                         dk(  sJ |j                  |k(  sJ t	        |j
                        |dik(  sJ y )NrA   rB   r   )r   r   r   r   r  r2   )r   r   new_keyrz   r   r   r  s          r5   test_persist_delayed_renamer    s     	c1XA99;!**,MGT	'1	4	4V	4B::<166W=WaL(((r7   c                     t        d      } t        |       \  }t        |t              sJ |j	                         dk(  sJ y r  r  )rC   rj  s     r5   test_persist_delayedleafr  (  s:    
AAJERb'"""::<1r7   c                      G d d      } t        |       j                  }t        |      \  }t        |t              sJ |j                         dk(  sJ y )Nc                      e Zd ZdZy)#test_persist_delayedattr.<locals>.CrA   N)r<   r=   r>   rC   r?   r7   r5   Cr  0  s    r7   r  rA   )r   rC   r   r   r   r   )r  rC   rj  s      r5   test_persist_delayedattrr  /  sJ      	
AAJERb'"""::<1r7   c                 b   t         j                  dd      dz   } t        j                  g d      j	                  t
              }t        j                  t              5  t        | |       d d d        t        | |d      \  }}t        |t         j                        sJ t        |t        j                        sJ |j                  | j                  k(  sJ |j                  |j                  k(  sJ t        |j                        |j                   cxk(  rt        | j                        k  sJ  J t        |j                        |j                   cxk(  rt        |j                        k  sJ  J t"        j%                  | |      sJ t'        |      t'        |      k(  sJ y # 1 sw Y   ;xY w)Nru   rB   r  rA   rG   r   r   )r  r  r   r   r   r$   rH   rf   r   r   r   r(  Bagr'  r   r2   r   r
  r  list)rC   rw   rj  rk  s       r5   test_persist_array_bagr  9  sK   
		!A	"A
#'',A	z	" 1 Q%67FBb"((###b"&&!!!77aff77affrww<2>>7CK77777rww<2>>7CK77777;;q"7d2h s   F$$F.c                 j   t        j                  g dd      j                  d       } t        | j	                               dk(  sJ | j                  d      }t        |t         j                        sJ t        |j	                               dk(  sJ t         j                  j                  | |       y )NrG   rB   r   c                    | dz  S NrB   r?   r  s    r5   r   z"test_persist_bag.<locals>.<lambda>P  
    Q r7   r   syncr   )
r   r   r   r   r:   r   r   r  r)  r  r.  s     r5   test_persist_bagr  O  s    
266GAq!"a'''			F	#Aa   q!"a'''HHq!r7   c                    t        j                  g dd      j                  d       j                         } t	        | j                               dk(  sJ | j                  d      }t        |t         j                        sJ t	        |j                               dk(  sJ t         j                  j                  | |       y )	NrG   rB   r   c                    | dz  S r  r?   r  s    r5   r   z#test_persist_item.<locals>.<lambda>Y  r  r7   r   r  r   rA   )r   r   r   rB  r   r:   r   r   Itemr)  r  r.  s     r5   test_persist_itemr  X  s    
266GKKMAq!"a'''			F	#Aa!!!q!"a'''HHq!r7   c                 h   t        j                  g dd      } | j                         \  }}dgddgd} ||g|d| j                  d	ii}t	        |t         j
                        sJ |j                  d	k(  sJ |j                         d
dgk(  sJ t         j                  j                  |g d       y )NrG   rB   r   r   ru   r7  r!  r   rw   r"  r#  )r   ru   r7  )	r   r   r   r'  r   r  rY   r)  r  r*  s        r5   test_persist_bag_renamer  a  s    
2A**,MGTSQF
+C1d1AFFC=1Aa   66S==??8 4444HHq)$r7   c                 L   t        j                  g d      j                         } | j                         \  }} |ddig|d| j                  dii}t        |t         j                        sJ |j                         dgk(  sJ t         j                  j                  |d       y )NrG   rC   r   r   )
r   r   rB  r   r'  r   r  rY   r)  r  )rn   r   r   rw   s       r5   test_persist_item_change_namer  l  s    
#'')A**,MGTa6466Aa!!!??%%%HHq!r7   c                    t        j                  d       t        j                  d      } | j                  dd      }d }ddlm}  ||d	z   t
        j                  d      d	z          t        j                  j                  |
      5   ||d	z   t
        j                  d      dz  d	z   dz  d       d d d         ||d	z   t
        j                  d      d	z          t        j                  dd      }t        j                  j                  |
      5  t        j                  |d	z   |j                  t              d      \  }} ||t
        j                  d      dz  d	z   dz         d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)Nr'   r%   ry   rn  r  c                d    | j                         D ci c]  \  }}|t        d|f c}}S c c}}w r  )itemsr   )r4   r   r  vs       r5   optimize_doublez.test_optimize_globals.<locals>.optimize_double{  s*    +.99;741aCA;777s   ,r   r  rA   )array_optimizerB   F)check_chunksr   r   r   )rH   rI   onesr  r  r
  r2   r  r   r   r   r   r   r$   )r  rC   r  r  rw   rj  rk  s          r5   test_optimize_globalsr  u  sB   
 			\	*B
4 A8 +a!eRWWR[1_%		8 H!a%"''"+/A-2GH a!eRWWR[1_%
#A		8 1a!eQUU3Z;LMB"rwwr{Q*a/01 1H H1 1s   	+E5AF5E>F
c                 8   t        j                  d       t        j                  d      } | j                  dd      }|d d dd d d d	   dz   fd
}t        j                  j                  d |      5  j                          d d d        y # 1 sw Y   y xY w)Nr'   r%   ry   rn  r  rH  rA   r   rB   c                d    | t        j                        k(  sJ t        j                  | |      S r0   )r  r2   r   )r4   r   r   s     r5   my_getz"test_optimize_None.<locals>.my_get  s*    d166l"""xxT""r7   )r  r   )rH   rI   r  r2   r  r   r   )r  rC   r  r   s      @r5   test_optimize_Noner    s    
 			\	*B
4 A	"1a
3Q3!A# 
	? 			  s   6BBc                     d } | t         d<    t        t              d      }	 |j                         dk(  sJ |j                  d      dk(  sJ t        j
                  j                  d      5  |j                         dk(  sJ 	 d d d        |j                         dk(  sJ t        j
                  j                  d      5  |j                  d      dk(  sJ 	 d d d        t         d= y # 1 sw Y   gxY w# 1 sw Y   xY w# t         d= w xY w)Nc                
    dggS )N{   r?   )r4   r   kwargss      r5   schedulez(test_scheduler_keyword.<locals>.schedule  s    wr7   r)   rA   rB   r   r  threads)r   r   r$   r   r2   r  r   )r  rC   s     r5   test_scheduler_keywordr    s     'UQA$yy{ayy5y)S000[[__u_- 	&99;#%%%	&yy{a[[__u_- 	799y91Q666	7 U#	& 	&	7 	7 U#s<   AD /C,=D C8D ,C51D 8D=D 	Dc                      t        t              d      } t        j                  t              5 }| j                  t        j                         d d d        dt        j                        v sJ y # 1 sw Y   #xY w)NrA   r   z
scheduler=)
r   r$   rH   rf   r   r   r2   r   strvalue)rC   infos     r5   test_raise_get_keywordr    s^    QA	y	!  T			dhh	  3tzz?***   s   !A22A;c                      e Zd ZdZy)
MyExecutorN)r<   r=   r>   _max_workersr?   r7   r5   r  r    s    Lr7   r  c                    t               J t        t        j                  j                        t        j                  j                  u sJ t        d      t        j                  j
                  u sJ t        d      t        j                  j                  u sJ t        t        t        j                  j                              sJ t        t        t                           sJ t        j                  j                  d      5  t               t        j                  j
                  u sJ 	 d d d        t               J y # 1 sw Y   xY w)Nr   r  r  )r   r2   localget_syncr   r   callablesynchronous_executorr  r  r   r?   r7   r5   test_get_schedulerr    s    ?"""4::#6#674::;N;NNNN9-1B1BBBB6*djj.A.AAAAMDJJ,K,KLMMMMJL9:::	9	- 4$--"3"33334?"""4 4s   'EEc                 p    dgfd}  t        d              j                  |       dk(  sJ d   sJ y )NFc                :    dd<   t        j                  | |      S )NTr   )r2   r   )r4   r   r   r  calleds       r5   r   z$test_callable_scheduler.<locals>.get  s    q	xxT""r7   c                      yr  r?   r?   r7   r5   r   z)test_callable_scheduler.<locals>.<lambda>  s    r7   r   rA   r   ru  )r   r  s    @r5   test_callable_schedulerr    sC    WF# 79''#'6!;;;!99r7   ry   ru   )rerunsreruns_delayr   r  	processesc                    t        d      t        j                        }d}t        |      D cg c]
  } |d       }}t        j
                  j                  |d      5  t               5 }t        |d| i d d d        d d d        j                  D ch c]  }|j                   }}t        |      |k(  sJ y c c}w # 1 sw Y   JxY w# 1 sw Y   NxY wc c}w )NF)purer*   g      ?rA   )num_workers	chunksizer   )r   timesleepr   r2   r  r   r   r   results	worker_idr   )r   r|   r  r   rn   profworkerss          r5   test_num_workers_configr     s     	UDJJ'A K{+,A3,A,	[A	> )
 )d(i() ) %)LL1qq{{1G1w<;&&& 	-) ) ) ) 2s/   B="C-C9CCC	CCc                 2   t        j                  d       t        j                  d      } | j                  dd      d d }t        |g      }t        j
                  j                  dd gi      5  t        |g      }d d d        |k(  sJ y # 1 sw Y   xY w)Nr'   r%   rB   rA   r  r  c                    | S r0   r?   )r4   r   s     r5   r   z(test_optimizations_ctd.<locals>.<lambda>  s    c r7   )rH   rI   r  r   r2   r  r   )r  rC   dsk1r   s       r5   test_optimizations_ctdr$    s    
 			\	*B
		!A	r"Aqc"D	/,A+BC	D '!1#&' 4<<' 's   0BBc                 >   t        dd      dk(  sJ t        dd      dk(  sJ t        dd      dk(  sJ t        dd      d	k(  sJ t        d
t        dfd      dt        dfk(  sJ t        j                  t              5  t        dd       d d d        y # 1 sw Y   y xY w)Nz	inc-1-2-3r  z$inc-73db79fdf4518507ddc84796726d4844rC   z"x-c4fb64ccca807af85082413d7ef01721i  "x-d4b538b4d4cf68fca214077609feebaerK  )r&  rA   z	sum-1-2-3rA   z$sum-822e7622aa1262cef988b3033c32aa37rB   )r   r   rH   rf   r   r?   r7   r5   test_clone_keyr'    s    [#&*PPPPS#"FFFFS#"FFFFXs#'PPPPk2q)3/.
	4   
 
y	! !Q  s   =BBc                 F   t        j                  d       t        j                  d      } | j                  d      }d G fdd| j                        }t	        ||j                         |j                               d   }sJ | j                  j                  ||       y )Nr'   r%   ry   Fc                  0     e Zd ZdZe fd       Z xZS )Gtest_compute_as_if_collection_low_level_task_graph.<locals>.MyDaskArrayz>Dask Array subclass with validation logic in __dask_optimize__c                P    t        |      t        u sJ dt        |   ||fi |S )NT)typer   super__dask_optimize__)clsr4   r   r  	__class__	optimizeds       r5   r.  zYtest_compute_as_if_collection_low_level_task_graph.<locals>.MyDaskArray.__dask_optimize__  s3     9...I7,S$A&AAr7   )r<   r=   r>   __doc__classmethodr.  __classcell__)r0  r1  s   @r5   MyDaskArrayr*    s    L		B 
	Br7   r5  r   )	rH   rI   r  r(  r   r:   rY   r)  r  )r  rC   r5  r   r1  s       @r5   2test_compute_as_if_collection_low_level_task_graphr6    s    
 			\	*B
		"A I	Bbhh 	B &Q%%'):	F 9HHq&!r7   c                :   ddl m} ddlm} ddlm} |r|j                  dd|i      }n |       }|5  dd l}|dk(  rddlm}	 n|dk(  rddlm	}	 n|d	k(  rddl
m	}	 |j                  |       }
t        |
|      j                  	k(  sJ 	 d d d        y # 1 sw Y   y xY w)
Nr   )nullcontext)mock)r
  zdask.base.named_schedulersr  r  r   r  )
contextlibr8  unittestr9  
dask.localr
  patch	importlibdask.threadedr   dask.multiprocessingimport_modulegetattrr   )module
collectionexpected
emscriptenr8  r9  r
  ctxr>  r   r  s              r5   check_default_schedulerrH  "  s    &#jj57IJm	 Bv2")$0%%f-sJ'::cAAAB B Bs   ABBparams)z('dask.dataframe', '_Frame', 'sync', Truez,'dask.dataframe', '_Frame', 'threads', Falsez#'dask.array', 'Array', 'sync', Truez''dask.array', 'Array', 'threads', Falsez'dask.bag', 'Bag', 'sync', Truez%'dask.bag', 'Bag', 'processes', Falsec                   t        j                  d       t        j                  d       t        j                  d       t        j                  d      }|j                         rd| v rt        j                  d       t	        j
                  t        j                  dt        j                  t              d|  dz   g      }|j                          y )	Nr'   r%   r(   r&   zobjects not availabler  zcheck_default_scheduler(z)
)rH   rI   r8  rL  r  runr  r  inspect	getsourcerH  check_returncode)rI  r0  procs      r5   !test_emscripten_default_schedulerrP  ;  s      
%
!			-	.B	#3v#=+,>>NN!!"9:,VHC89		
	D 	r7   )
__future__r   r   rL  r  r  r  r  r   r   concurrent.futuresr   operatorr   r   typingr	   rH   tlzr
   r   r2   dask.bagbagr   	dask.baser   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   dask.delayedr   r   dask.diagnosticsr   dask.highlevelgraphr   
dask.utilsr    r!   dask.utils_testr"   r#   r$   r  r0  r
  rL   r   r+   rD   rP   ri   r   r   r   r   r   r   r  markskipifr  r  r+  r4  r9  r>  rE  rQ  rU  rW  r[  re  rg  rl  rq  rw  boolflagsxfailplatformr  r  r  r  r  r  r  r  r  r  r  parametrizer  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  flakyslowr   r$  r'  r6  rH  rP  r?   r7   r5   <module>rg     s   "   	  
  # '          $ * % . & 4 4L!$%GH 	3#$8=@B	 &$#  # LK!\<6 H& & H
( 
( H( ( H* * H# # H# # H
# 
# H" "  H" "  H    &') () H& &
 H& & H  H  H		%N   LLGG  
&# &#R H		%N  1 1 H		%N  " ",:!."6)! 	( wm 45% 6% 	b#	sCj#	sCj#
2'*WbM:	)) H *%12$,+ 	# "1-y+&>?' @  .' ">B2 

r7   