
    tf#                    >   d dl mZ d dlZd dlZ ej                  d       d dlZd dlmZ d dlm	Z	m
Z
 d dlmZmZ d dlmZ d dlmZmZmZ  G d	 d
e
      Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zej@                  jC                  d      ej@                  jE                  dedfedfedfedfedfedfedfedfg      ej@                  jE                  dddg      d                      Z#d Z$d  Z%ej@                  jE                  d!e$e%g      d"        Z&d# Z'y)$    )annotationsNdistributed)getitem)ClientSchedulerPlugin)clusterloop)HighLevelGraph)ArrayChunkShapeDepArraySliceDepfractional_slicec                       e Zd ZdZdZd Zd Zy)SchedulerImportCheckzAPlugin to help record which modules are imported on the schedulerzimport-checkc                    || _         y N)pattern)selfr   s     `/var/www/html/software/conda/envs/higlass/lib/python3.12/site-packages/dask/tests/test_layers.py__init__zSchedulerImportCheck.__init__   s	        c                  K   t               | _        t        t        j                        D ]X  }|j	                  | j
                        s| j                  j                  |       :t        j                  j                  |       Z y wr   )setstart_modulessysmodules
startswithr   addpop)r   	schedulermods      r   startzSchedulerImportCheck.start   s^      Us{{# 	%C>>$,,/""&&s+ $	%s   BBN)__name__
__module____qualname____doc__namer   r!    r   r   r   r      s    KD%r   r   c                 &   t        j                  d       t        j                  d      } ddt        fdD              }| j                  |      }t	        |      fd}t              D ]  }t              D ]  } |||         y )Nnumpydask.array.core   r+      c              3  (   K   | ]	  }|z    y wr   r'   .0nds     r   	<genexpr>z-test_array_chunk_shape_dep.<locals>.<genexpr>+        'A!a%'   c                "    | |f   }|k(  sJ y r   r'   )ijchunk_shape
array_depschunks      r   checkz)test_array_chunk_shape_dep.<locals>.check/   s     !Q(e###r   )pytestimportorskiptuplenormalize_chunksr   range	dacshapechunksr<   r7   r8   r:   r;   r2   s	         @@@r   test_array_chunk_shape_deprF   &   s    
 


/
0C	AE'''E!!%/F#F+J$ 1X q 	A!QK	r   c                 &   t        j                  d       t        j                  d      } ddt        fdD              }| j                  |      }t	        |      fd}t              D ]  }t              D ]  } |||         y )Nr)   r*   r+   r,   c              3  (   K   | ]	  }|z    y wr   r'   r/   s     r   r3   z(test_array_slice_deps.<locals>.<genexpr>=   r4   r5   c                    | |f   }|t        d   | z  d   | dz   z  d       t        d   |z  d   |dz   z  d       fk(  sJ y )Nr      )slice)r7   r8   slicesr:   r;   s      r   r<   z$test_array_slice_deps.<locals>.checkA   sj    QF#%(Q,aAE 2D9%(Q,aAE 2D9
 
 	
 
r   )r=   r>   r?   r@   r   rA   rB   s	         @@@r   test_array_slice_depsrM   8   s    
 


/
0C	AE'''E!!%/Fv&J
 1X q 	A!QK	r   c                    t        j                  d      }t        j                  d      }|j                  t        d      t        dd      d      }|j	                  |d      j                  dd	
      S )Npandasdask.dataframe
      abr+   npartitionsrT   tasksshuffle_method)r=   r>   	DataFramerA   from_pandasshuffletmpdirpddddfs       r   _dataframe_shufflerc   M   se    			X	&B			-	.B 
E"IE"bM:	;B>>"!>,44S4QQr   c                    t        j                  d      }t        j                  d      }|j                  t        d      t        dd      d      }|j	                  |d      j                         S )NrO   rP   rQ   rR   rS   r+   rV   )r=   r>   r[   rA   r\   meanr^   s       r   _dataframe_tree_reductionrf   V   s^    			X	&B			-	.B 
E"IE"bM:	;B>>"!>,1133r   c                   t        j                  d      }t        j                  d      }|j                  t        d      t        dd      d      }|j	                  |d      }|j	                  |d      }|j                  |d	d
d      S )NrO   rP   rQ   rR   rS      rV   rJ   leftTrX   )how	broadcastrZ   )r=   r>   r[   rA   r\   merge)r_   r`   ra   rb   ddf1ddf2s         r   _dataframe_broadcast_joinro   _   s}    			X	&B			-	.B 
E"IE"bM:	;B>>"!>,D>>"!>,D::d$w:OOr   c                r    t        j                  d      }|j                  d      |j                  d      z   S )N
dask.arrayd   )r=   r>   oneszeros)r_   das     r   _array_creationrw   j   s0    			\	*B 776?RXXf---r   c                x    t        j                  d      }|j                  d      }|j                  d dd      S )Nrq   rr   c                    | S r   r'   xs    r   <lambda>z$_array_map_overlap.<locals>.<lambda>t   s    q r   rJ   none)depthboundary)r=   r>   rt   map_overlap)r_   rv   arrays      r   _array_map_overlapr   q   s7    			\	*BGGFOE[FCCr   c                 `   t        dddi      t        dt        dd      ffk(  sJ t        dddd      t        dt        d d d       t        d	d       ffk(  sJ t        d
ddd      t        dt        dd      t        d	d       ffk(  sJ t        dddi      } t        | d   d   t              sJ y )N)r{   g@r   r+   )r{      )r{   r-   ffffff@r-   )r   rJ   )r{   r-   r   )r{   g333333@r   rJ   )r   r   rK   
isinstanceint)fss    r   test_fractional_slicer   w   s    JA/GXaQR~3VVVVMqQ<8	tT4	 %D/2=    Oq\:	q!eBo&?    
*q!f	-BbeAh$$$r   c                L   t        j                  d       t        j                  d      }t        j                  d      }|j                  |j                  dt	        d      i      d      j                  t        |              dg}|j                  t        |       |	      }|S )
NpyarrowrO   rP   rT   rQ   r+   rV   )rT   z<=r+   )filters)r=   r>   r\   r[   rA   
to_parquetstrread_parquet)r_   r`   ra   r   rm   s        r   _pq_pyarrowr      s    
	"			X	&B			-	.BNN2<<eBi 01qNALLF  G??3v;?8DKr   c                   t        j                  d       t        j                  d      }t        j                  d      }|j                  |j                  dt	        d      i      d      }t        j
                  t              5  |j                  t        |       d       |j                  t        |       d      cd d d        S # 1 sw Y   y xY w)	NfastparquetrO   rP   rT   rQ   r+   rV   )engine)
r=   r>   r\   r[   rA   warnsFutureWarningr   r   r   r^   s       r   _pq_fastparquetr      s    
&			X	&B			-	.B	c59%56A	FB	m	$ B
c&k-8s6{=AB B Bs   8CCc                P   t        j                  d      }t        j                  d      }|j                  |j                  dt	        d      i      d      j                  t        |              |j                  t        j                  j                  t        |       d            S )NrO   rP   rT   rQ   r+   rV   *)r=   r>   r\   r[   rA   to_csvr   read_csvospathjoin)r_   r`   ra   s      r   	_read_csvr      s~    			X	&B			-	.BNN
c59%&   fS[;;rww||CK566r   z#8480)reasonzop,libzpandas.znumpy.optimize_graphTFc                   t        dt              gi      5 \  }}t        |d   |      5 }|j                   | |      |       |j	                  d       }|j	                  d       }	||	z
  }
t        fd|	D              rJ t        fd	|
D              rJ 	 d d d        d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)
Nplugins)scheduler_kwargsaddress)r	   )r   c                 4    t        t        j                        S r   )r   r   r   r'   r   r   r|   z>test_scheduler_highlevel_graph_unpack_import.<locals>.<lambda>   s    S5E r   c                P    | j                   t        j                     j                  S r   )r   r   r&   r   )dask_schedulers    r   r|   z>test_scheduler_highlevel_graph_unpack_import.<locals>.<lambda>   s"    ~'='=(--(- r   c              3  @   K   | ]  }|j                          y wr   r   r0   modulelibs     r   r3   z?test_scheduler_highlevel_graph_unpack_import.<locals>.<genexpr>   s     Nf6,,S1N   c              3  @   K   | ]  }|j                          y wr   r   r   s     r   r3   z?test_scheduler_highlevel_graph_unpack_import.<locals>.<genexpr>   s     Lf6,,S1Lr   )r   r   r   computerun_on_schedulerany)opr   r   r	   r_   r   workerscend_modulesr   new_moduless    `         r   ,test_scheduler_highlevel_graph_unpack_importr      s    & 
9/CC/H.I"J	K M PIi(t4 	MIIbjI@ ,,-EFK.. M
 &5K NNNNN LLLLLL%	M	M M	M 	M	M Ms#   B;A/B/B;/B8	4B;;Cc                (    | j                  dd      S )Nr{   rX   rY   )r]   ddfs    r   _shuffle_opr      s    ;;s7;33r   c                F    | j                  d      j                  ddi      S )Nr&   r{   re   )groupbyaggr   s    r   _groupby_opr      s     ;;v""C=11r   r   c                   t        j                  d       t        j                  d      }|j                         rt        j                  d       t        j                  d      } | |j	                  d            j                         }|j                  }|j                  |j                               }|j                         |j                         k(  sJ y )NrO   rP   not supporteddask.datasets
2000-01-15end)
r=   r>   _dask_expr_enabledskip
timeseriescountdaskcull__dask_keys__get_all_dependencies)r   ra   datasetsresultgraphculled_graphs         r   $test_dataframe_cull_key_dependenciesr      s     !			-	.B	O$""?3H###56<<>FKKE::f2245L%%'<+L+L+NNNNr   c                    t        j                  d       t        j                  d      } t        j                  d      }|j                         rt        j                  d       | j	                  d      }d}d}i }t        |j                        D ]$  }d	 |j                  |ff|||f<   d
 ||ff|||f<   & t        j                  |||g      }|j                  j                  |||j                  |j                        }|j                  }|j                         D 	cg c]  }	|	|dfk7  s|	 }
}	|j!                  |
      }|j"                  j%                         }|j'                         }||k(  sJ |j'                         }t)        |j+                               D ]  \  }}|dk(  s|j-                  ||f         ||k(  sJ y c c}	w )NrO   r   rP   r   r   r   custom_graph_testcustom_graph_test_0c                    | S r   r'   rz   s    r   r|   zCtest_dataframe_cull_key_dependencies_materialized.<locals>.<lambda>  s    a r   c                    | S r   r'   rz   s    r   r|   zCtest_dataframe_cull_key_dependencies_materialized.<locals>.<lambda>  s    A r   )dependenciesr   )r=   r>   r   r   r   rA   rW   _namer
   from_collectionscorenew_dd_object_meta	divisionsr   r   r   key_dependenciescopyr   listkeysr   )r   ra   r   r&   name_0dskr7   r   r   kculled_keysr   cached_depsdepsdeps0s                  r   1test_dataframe_cull_key_dependencies_materializedr      s    !""?3H			-	.B	O$


,

/C D"F
C3??# 4'#))Q8VQK%{3T1I4 
)
)$3%
HCWW""3ciiGFKKE %224GdAY1GKG::k*L //446K,,.D$ &&(E

% !a6IItQi !
 D==! Hs   "G	/G	)(
__future__r   r   r=   r>   r   operatorr   r   r   r   distributed.utils_testr   r	   dask.highlevelgraphr
   dask.layersr   r   r   r   rF   rM   rc   rf   ro   rw   r   r   r   r   r   markxfailparametrizer   r   r   r   r   r'   r   r   <module>r      s\   " 	    M " 
  / 0 . K K%? %&$*R4P.D%&B7 '"	Y'	"I.	"I.	i 	)$	I	(#	X&	 )D%=9M : #M:42 [9:O ;O$)r   