
    >ie:              	         d dl mZ d dlZd dlZd dlmZ d dlZd dlZd dlm	Z
 d dlmZ d dlmZ d dlmZ d dlmZmZmZmZmZ d dlmZ d d	lmZ d d
lmZ  ed          Z ed          Z ed          Z ed          Z G d d          Z d3d4dZ!d Z"dZ#dZ$d Z%d5dZ&ej'        (                    dddg          d             Z)ej'        *                    d          d              Z+ej'        (                    dddg          d!             Z,ej'        (                    dddg          d"             Z-ej'        *                    d          d#             Z.ej'        (                    dddg          d$             Z/ej'        *                    d%          ej'        (                    d&d'd(d'gd'd'id'hg          d)                         Z0ej'        *                    d*          d+             Z1ej'        (                    dddg          d,             Z2ej'        *                    d          ej'        (                    d-eeg          d.                         Z3ej'        (                    d/g d0          d1             Z4d2 Z5dS )6    )annotationsN)add)delayed)	clone_key)	Blockwise)bind
checkpointchunksclonewait_on)HighLevelGraph)Tuple)import_or_nonez
dask.arrayzdask.dataframepandaszarrc                      e Zd Zd Zd ZdS )NodeCounterc                    d| _         d S )Nr   )n)selfs    Blib/python3.11/site-packages/dask/tests/test_graph_manipulation.py__init__zNodeCounter.__init__   s        c                v    t          j        t          j                    dz             | xj        dz  c_        |S )Nd      )timesleeprandomr   )r   xs     r   fzNodeCounter.f   s1    
6=??S()))!r   N)__name__
__module____qualname__r   r!    r   r   r   r      s2              r   r   layersboolreturnNonec                  |                                  }|                                 }|#|                                 }|                                |                                z
  |                                z  rJ |                                |                                z
  rJ |                                |                                z
  rJ |rr|j                                        |j                                        z
  |j                                        z  rJ |j                                        |j                                        z
  |j                                        z  rJ |j                                        |j                                        z
  rJ |j                                        |j                                        z
  rJ |j                                        |j                                        z
  rJ |j                                        |j                                        z
  rJ d S d S |                                |                                z  rJ |rj|j                                        |j                                        z  rJ |j                                        |j                                        z  rJ d S d S N)__dask_graph__keysr&   dependencies)abomitr&   dsk1dsk2dskos          r   assert_no_common_keysr5   #   s   DD""$$IIKK$))++-<<<<99;;,,,,99;;,,,, 		K((**T[-=-=-?-??4;CSCSCUCUUUUU&++--0A0F0F0H0HH#((**+   {''))DK,<,<,>,>>>>>{''))DK,<,<,>,>>>>>(--//$2C2H2H2J2JJJJJ(--//$2C2H2H2J2JJJJJ		K 		K KJ 99;;,,,, 	K{''))DK,<,<,>,>>>>>(--//$2C2H2H2J2JJJJJ	K 	KJJr   c                   |                                 j        }|                                  j        }|                                D ]v\  }}	 ||         }n&# t          $ r |t	          |d                   }Y nw xY wt          |t                    r,|                                rJ |                                rJ wdS )zTest that all layers of the original collection exist in the cloned collection
    too and that Blockwise layers have not been materialized
    r   N)r,   r&   itemsKeyErrorr   
isinstancer   is_materialized)clonedorigolayersclayerskvcvs          r   assert_did_not_materializerB   >   s     !!##*G##%%,G , ,1	*BB 	* 	* 	*1a)BBB	*a## 	,((*****))+++++, ,s   A A98A9)g333333?foor%   h2c                    t                      } t                              dt          t	          d                    i          } t          | j                  d          t                              dd          	                    | j                  t                              dd          	                    | j                  t          j        d	d
gd
                              | j                  t          j        d	gd	                              | j                  t          j                             t          | j                  d                    t                              |d
                              | j                  t                              |d	                              | j                  t                              |d         d
                              | j                  t                              |d         d	                              | j                  g
}d| _        || fS )Nr    
   zHello 1rF   rF      r
   r   r      npartitionszHello 2r   )r   pd	DataFramelistranger   r!   daones
map_blocksdbfrom_sequencemapItemfrom_delayedddfrom_pandasmap_partitionsr   )cntdfcollss      r   collections_with_node_countersra   S   s   
--C	sDrOO,	-	-B 	y!!
##..su55
R  ++CE22
!QQ///33CE::
!!,,,0077
^WSU^^I6677
rq))88??
rq))88??
r#wA..==ceDD
r#wA..==ceDDE  CE#:r    tuple[Tuple, Tuple, NodeCounter]c                   t                      }t          dt          f|j        dfdt          f|j        dfid|j        dfidt                      t                      d          }t          d|j        dfid	|j        d
fidt                      t                      d          }| s(|                                }|                                }t          |t          |                    t          |t          |                    |fS )Nr/   r   rL   r0      r/   r0   c   drH   rf   rh   )	r   r   h1r!   rD   setto_dictr   rQ   )r&   r^   r2   r3   s       r   demo_tuplesrm   m   s    
--CRy35!*sBi#%<C#%QRCTUUee#%%   D SUAJsSUAJ&788ee#%%   D  ||~~||~~tDzz""E$T

$;$;S@@r   FTc                    t          |           \  }}}t          |d|gi          }|                    d          J |j        dk    sJ d S )Nr    sync	schedulerrH   )rm   r	   computer   )r&   t1t2r^   cps        r   test_checkpointrv      sW    f%%KBC	Brd	$	$B:::''///5A::::::r   znot da or not ddc                     t                      \  } }t          |  }|                    d           |j        dk    sJ d S )Nro   rp      )ra   r	   rr   r   )r`   r^   ru   s      r   test_checkpoint_collectionsry      sB    /11JE3	U	BJJJ   5B;;;;;;r   c                    t          |           \  }}}t          |          }|                    d          dk    sJ |j        dk    sJ d S )Nro   rp   r   rL   rd   rd   )rm   r   rr   r   )r&   rs   _r^   t1ws        r   test_wait_on_oner~      sR    V$$JB3
"++C;;;((I55555A::::::r   c                    t          |           \  }}}t          |d|gi          }t          j        |ddidddgifk    sJ |j        dk    sJ d S )Nr    rq   ro   r{   )rg   rH   rH   )rm   r   daskrr   r   )r&   rs   rt   r^   outs        r   test_wait_on_manyr      sh    f%%KBC
"sRDk
"
"C<///IfX3OOOOO5A::::::r   c                 B   t                      \  } }t          d             }t          |  } ||d                                                    |j        dk    sJ |d                                         | d                                         k    sJ t
          j                            |d         | d                    t
          j                            |d         | d                    t          j                            |d         | d                    t          j                            |d         | d                    t          j                            |d         | d                    t          j                            |d	         | d	                    t          j                            |d
         | d
                    t          j                            |d         | d                    t          j                            |d         | d                    d S )Nc                    d S r+   r%   r    s    r   r!   z#test_wait_on_collections.<locals>.f   s    r   r   rx   r   rL   rd   rg   rH            	   )
ra   r   r   rr   r   rS   utils	assert_eqrV   r[   )r`   r^   r!   colls2s       r   test_wait_on_collectionsr      s   /11JE3   W e_FAfQiLL5B;;;; !9%("2"2"4"44444Hvay%(+++Hvay%(+++Hvay%(+++Hvay%(+++Hvay%(+++Hvay%(+++Hvay%(+++Hvay%(+++Hvay%(+++++r   c           
        dt           fddt          fdi}dt          dt           fdt          ffi}ddd}| rOt          j        d|          }t          ||dt                      dhd          }t          j        d|          }n|                    |           t          |dt           fdt          fg          }t          |dg          }t          |dg          }t          |d| 	          }t          |d| 	          }t          |d| 	          }	t          |d| 	          }
t          || 
          }t          || 
          }t          ||d|           }|	                                |	                                k    sJ t          |||            t          ||	|            t          ||
|            t          |||            t          |||            t          ||||            t          j        ||||	|
|||          dk    sJ t          d|gid||gi|           }t          j        |          ddgifk    sJ |d         d         }t          ||||            t          |||            d S )Nr/   r   rL   r0   ri   re   )r.   rf   seedassume_layers)r   r1   r   r   r&   r1   r&   )rd   r   r   r   r   r   r   r   r    y)r1   r   r   r   )rj   rD   r   r   from_collectionsrk   updater   r   r,   r5   r   rr   )r&   r2   r3   dsk3rs   rt   t3c1c2c3c4c5c6c7r   c8s                   r   
test_cloner      s   "Iq3)Q'D#Ry3),-DD .sD99T""suuC51I1I
 
 
 .sD99D	tsBi#r+	,	,B	tcU		B	tcU		B	r	0	0	0B	r	0	0	0B	r	0	0	0B	r	0	0	0B	r	(	(	(B	r	(	(	(B	r&	9	9	9B""3"3"5"55555"b0000"b0000"b0000"b0000"b0000"br&9999<BBBB77;FFFF rd3R/
H
H
HC<#v 00000	S!B"br&9999"b000000r   znot daliteralr   rJ   c                   t                               dd          }d }t                               |d|d| d          }t          |          }t	          ||d           t           j                            ||           dS )	zhttps://github.com/dask/dask/issues/8978

    clone() on the result of a dask.array.blockwise operation with a (iterable) literal
    argument
    rF   r   rI   c                    | S r+   r%   )arrlits     r   noopz0test_blockwise_clone_with_literals.<locals>.noop   s    
r   r    NTr   )rS   rT   	blockwiser   r5   r   r   )r   r   r   blkclns        r   "test_blockwise_clone_with_literalsr      s    " ''"Q'

C   ,,tS#sGT
:
:C
**C#s40000HsC     r   znot da or not zarrc                    t                               t                              d                    } t	          | j        j        | j                 t                    sJ t          d | j        D                       sJ t          |           }t          | |d           t           j                            | |           dS )z|https://github.com/dask/dask/issues/9621

    clone() on a Blockwise layer on top of a dependency layer with no indices
    rF   c              3  @   K   | ]}t          |t                    V  d S r+   )r9   str).0r?   s     r   	<genexpr>z7test_blockwise_clone_with_no_indices.<locals>.<genexpr>  s,      44az!S!!444444r   Tr   N)rS   	from_zarrr   rT   r9   r   r&   namer   anyr   r5   r   r   )r   r   s     r   $test_blockwise_clone_with_no_indicesr     s     ,,tyy}}
%
%C choch/;;;;;443844444444
**C#s40000HsC     r   c                ^   dt           fddt          fdi}dt          dt           fdt          ffi}ddi}t                      }dt           f|j        dfdt          f|j        dfi}d|j        dfi}| rt          d|idt                      i          }t          ||d	t                      dhd	          }t          |||d
t                      dhdhd
          }t          ||dt                      t                      d          }n?|                    |           |                    |           |                    |           t          |dg          }t          |dg          }t          |dt           fdt          fdg          }	t          ||	d|           }
t          dd          }|
                                |t           f         d         t          j	        u sJ |
                                |t          f         d         t          j	        u sJ |
                                dk    sJ |j        dk    sJ t          ||	|d|           }t          dd          }|                                |         d         t          j	        u sJ |                                dk    sJ |j        dk    sJ t          |	|d|           }t          dd          }t          dd          }|                                |t           f         d         t          j	        u sJ |                                |t          f         d         t          j	        u sJ |                                |         d         t          j	        u sJ |                                dk    sJ |j        dk    sJ d S )Na-1r   rL   b-1c-1d-1erd   )r   r   )r   r   r   )r   r   r   )r   r   r   r   r   r{   r   )rj   rD   r   r   r!   r   rk   r   r   r   r   r,   r
   rr   r   )r&   r2   r3   r   r^   dsk4dsk4brt   r   t4bound1cloned_a_namebound2cloned_c_namebound3cloned_d_namecloned_e_names                    r   	test_bindr     s   BKUBK+DC%ubk23D5>D
--CBK#%eR[35!*=D35!*E udmeSUU^<<&&ug(F(F
 
 d33EE5'5'::
 
 d77SUU9S9STTDDE 
teW		B	teW		B	tubkE2;4	5	5B"bq777Fe!,,,M  ""="#45a8FKGGGG  ""="#45a8FKGGGG>>t####5A::::"br@@@Fe!,,,M  ""=1!4CCCC>>t####5A::::"bq777Fe!,,,Mc***M  ""="#45a8FKGGGG  ""="#45a8FKGGGG  ""=1!4CCCC>>y((((5A::::::r   funcc                   t           d             } |d          } ||          }t                              dd          }|dz   }|j        }t	          j        ddgd          }|                    d           }|                    d	           }	|	                                }
t          	                    d
t          t          d                    i          }t                              |d          }|                    d           }|                    d           }|d
         }|                                }t                      }| t           u rbt                              dd                              |j                  }d|_        t!          |||	|
|||f|||||fd          \  }}}}}}}n$t)          |||	|
|||||||fd	  	        \  }}}}}}}t+          ||           t+          ||           t+          ||	           t+          ||
           t+          ||           t+          ||           t+          ||           t-          |||d           t-          |||d           t-          ||	|d           t-          |||d           t-          |||d           t-          |||d           |                                |                                k    sJ |j        dk    s| t(          u sJ t          j                            ||           |j        dk    s| t(          u sJ t          j                            ||	           |j        dk    s| t(          u sJ t          j                            ||
           |j        dk    s| t(          u sJ t          j                            ||           |j        dk    s| t(          u sJ t          j                            ||           |j        dk    s| t(          u sJ t          j                            ||           |j        dk    s| t(          u sJ d S d S )Nc                    | dz  S NrL   r%   r   s    r   doublez+test_bind_clone_collections.<locals>.doubleN  s    1ur   rL   rG   rH   rI   r   rM   c                    | dz  S r   r%   r   s    r   <lambda>z-test_bind_clone_collections.<locals>.<lambda>\  
    !a% r   c                    | dz   S Nr   r%   r   s    r   r   z-test_bind_clone_collections.<locals>.<lambda>]  r   r   r    rF   c                    | dz  S r   r%   r   s    r   r   z-test_bind_clone_collections.<locals>.<lambda>c  
    Q r   c                    | dz   S r   r%   r   s    r   r   z-test_bind_clone_collections.<locals>.<lambda>d  r   r   r   )childrenparentsr1   r   )r1   r   Tr   rg   r      rx          $   )r   rS   rT   TrV   rW   rX   minrO   rP   rQ   rR   r[   r\   r]   r   r   rU   r!   r   r   rB   r5   rr   r   r   )r   r   d1d2a1a2a3b1b2b3b4r_   ddf1ddf2ddf3ddf4ddf5r^   parentd2ca3cb3cb4cddf3cddf4cddf5cs                             r   test_bind_clone_collectionsr   K  sG      W 
B	B	!	$	$B	aB	B		1a&a	0	0	0B		 	 B		 	 B	B	sDrOO,	-	-B>>"!>,,D//D//D9D88::D
--Ct||!,,77>>26"b"dD$7b"d#	3
 3
 3
/S#sE5%% 38b"d#
3
 
3
 
3
/S#sE5% sB'''sB'''sB'''sB'''ud+++ud+++ud+++#r48888#r48888#r48888%D>>>>%D>>>>%D>>>>::<<3;;==((((5A::HsB5A::HsB5B;;$%----HsB5B;;$%----Hud###5B;;$%----Hud###5B;;$%----Hud###5B;;$%----;;--r   zsplit_every,nkeys))rL   i+  )rd      )r      )Nr   )g333333 @r   )g    eA   )Fr   c                   d t          d          D             }t          |t          |                    }t          ||           }t	          |                                          |k    sJ |                    d          J t          ||           }t	          |                                          |dz   k    sJ |                    d          t          t          d                    k    sJ ddd}t          |t          |                    }t          ||| d	
          }t	          |                                          |dz   k    sJ |                    d          dk    sJ d S )Nc                    i | ]}d |f|S )r/   r%   )r   is     r   
<dictcomp>z$test_split_every.<locals>.<dictcomp>  s    
+
+
+1C8Q
+
+
+r   r   split_everyro   rp   r   rL   )r0   rf   F)r   r   )r   rL   )
rR   r   rQ   r	   lenr,   rr   r   tupler   )	r   nkeysdskrs   rf   rt   r   r   r   s	            r   test_split_everyr    sq    ,
+c


+
+
+C	sDII		B2;///Aq!!""e++++99v9&&...		-	-	-Br  ""##us{2222:::''5s+<+<<<<<D	tT$ZZ	 	 B	b"+U	C	C	CBr  ""##uqy0000:::''6111111r   c                    t          dddddg          } t          j        t                    5  t	          | d           d d d            n# 1 swxY w Y   t          j        t                    5  t	          | d           d d d            n# 1 swxY w Y   t          j        t                    5  t	          | d           d d d            n# 1 swxY w Y   t          j        t                    5  t	          | d	           d d d            n# 1 swxY w Y   t          j        t
                    5  t	          | ddi           d d d            d S # 1 swxY w Y   d S )
Nr   rL   re   r/   r0   r   gffffff?r   )r   pytestraises
ValueErrorr	   	TypeError)ts    r   test_split_every_invalidr	    s   AAc
++A	z	"	" % %1!$$$$% % % % % % % % % % % % % % %	z	"	" ' '1#&&&&' ' ' ' ' ' ' ' ' ' ' ' ' ' '	z	"	" % %1!$$$$% % % % % % % % % % % % % % %	z	"	" & &1"%%%%& & & & & & & & & & & & & & &	y	!	! * *11a&))))* * * * * * * * * * * * * * * * * *sY   AAA1BBB3CCC5DDD7EEEr+   )r&   r'   r(   r)   )r&   r'   r(   rb   )6
__future__r   r   r   operatorr   r  r   dask.bagbagrV   r   	dask.baser   dask.blockwiser   dask.graph_manipulationr   r	   r
   r   r   dask.highlevelgraphr   dask.tests.test_baser   dask.utils_testr   rS   r[   rO   r   r   r5   rB   rj   rD   ra   rm   markparametrizerv   skipifry   r~   r   r   r   r   r   r   r   r  r	  r%   r   r   <module>r     sT   " " " " " "                             $ $ $ $ $ $ L L L L L L L L L L L L L L . . . . . . & & & & & & * * * * * *^L!!^$%%^H~f       K K K K K6, , ," 	  4A A A A$ E4=11  21 &''  (' E4=11  21 E4=11  21 &'', , (',8 E4=11'1 '1 21'1T H		
	
A	
	 	! !	 	 !& ())! ! *)!" E4=110 0 210f &''$//N( N( 0/ ('N(b    2 2 2$* * * * *r   