o
    Nrf#                     @  s~  d dl mZ d dlZd dlZed d dlZd dlmZ d dlm	Z	m
Z
 d dlmZmZ d dlmZ d dlmZmZmZ G d	d
 d
e
Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd  Zej j!d!d"ej "d#ed$fed$fed$fed$fed$fed$fed%fed%fgej "d&d'd(gd)d* Z#d+d, Z$d-d. Z%ej "d/e$e%gd0d1 Z&d2d3 Z'dS )4    )annotationsNdistributed)getitem)ClientSchedulerPlugin)clusterloop)HighLevelGraph)ArrayChunkShapeDepArraySliceDepfractional_slicec                   @  s$   e Zd ZdZdZdd Zdd ZdS )SchedulerImportCheckzAPlugin to help record which modules are imported on the schedulerzimport-checkc                 C  s
   || _ d S N)pattern)selfr    r   _/var/www/html/software/conda/envs/catlas/lib/python3.10/site-packages/dask/tests/test_layers.py__init__   s   
zSchedulerImportCheck.__init__c                   sD   t  | _t tjD ]}|| js| j| q
tj| q
d S r   )setstart_modulessysmodules
startswithr   addpop)r   	schedulermodr   r   r   start   s   zSchedulerImportCheck.startN)__name__
__module____qualname____doc__namer   r   r   r   r   r   r      s
    r   c                    ~   t d t d} ddtfddD }| |}t|  fdd}tD ]}tD ]}||| q4q.d S )	Nnumpydask.array.core   r&      c                 3      | ]} | V  qd S r   r   .0ndr   r   	<genexpr>+       z-test_array_chunk_shape_dep.<locals>.<genexpr>c                   s    | |f }|ksJ d S r   r   )ijZchunk_shape
array_depschunkr   r   check/   s   z)test_array_chunk_shape_dep.<locals>.check)pytestimportorskiptuplenormalize_chunksr
   rangeZdacshapechunksr6   r1   r2   r   r4   r5   r.   r   test_array_chunk_shape_dep&   s   

r@   c                    r#   )	Nr$   r%   r&   r'   c                 3  r)   r   r   r*   r-   r   r   r/   =   r0   z(test_array_slice_deps.<locals>.<genexpr>c                   sX    | |f }|t d |  d | d  d t d | d |d  d fks*J d S )Nr      )slice)r1   r2   Zslicesr3   r   r   r6   A   s
   z$test_array_slice_deps.<locals>.check)r7   r8   r9   r:   r   r;   r<   r   r?   r   test_array_slice_deps8   s   

rC   c                 C  sF   t d}t d}|tdtddd}|j|ddjdd	d
S )Npandasdask.dataframe
      abr&   npartitionsrI   tasksshuffle_method)r7   r8   	DataFramer;   from_pandasshuffletmpdirpddddfr   r   r   _dataframe_shuffleM   s   

rX   c                 C  s@   t d}t d}|tdtddd}|j|dd S )NrD   rE   rF   rG   rH   r&   rK   )r7   r8   rP   r;   rQ   meanrS   r   r   r   _dataframe_tree_reductionV   s   

rZ   c                 C  s\   t d}t d}|tdtddd}|j|dd}|j|dd}|j|d	d
ddS )NrD   rE   rF   rG   rH      rK   rA   leftTrM   )how	broadcastrO   )r7   r8   rP   r;   rQ   merge)rT   rU   rV   rW   ddf1Zddf2r   r   r   _dataframe_broadcast_join_   s   

ra   c                 C  s   t d}|d|d S )N
dask.arrayd   )r7   r8   onesZzeros)rT   dar   r   r   _array_creationj   s   
rg   c                 C  s(   t d}|d}|jdd dddS )Nrb   rc   c                 S     | S r   r   xr   r   r   <lambda>t       z$_array_map_overlap.<locals>.<lambda>rA   none)depthboundary)r7   r8   re   Zmap_overlap)rT   rf   arrayr   r   r   _array_map_overlapq   s   

rq   c                  C  s   t ddditdtddffksJ t ddddtdtd d d td	d ffks*J t d
dddtdtddtd	d ffksAJ t dddi} t| d d tsSJ d S )N)rj   g@r   r&   )rj      )rj   r(   ffffff@r(   )r   rA   )rj   r(   rr   )rj   g333333@rs   rA   )r   r   rB   
isinstanceint)fsr   r   r   test_fractional_slicew   s   $

rx   c                 C  s`   t d t d}t d}|j|dtdiddt|  dg}|jt| |d	}|S )
NZpyarrowrD   rE   rI   rF   r&   rK   )rI   z<=r&   )filters)r7   r8   rQ   rP   r;   
to_parquetstrread_parquet)rT   rU   rV   ry   r`   r   r   r   _pq_pyarrow   s   


r}   c                 C  s   t d t d}t d}|j|dtdidd}t t |jt| dd |j	t| ddW  d    S 1 s?w   Y  d S )	NZfastparquetrD   rE   rI   rF   r&   rK   )Zengine)
r7   r8   rQ   rP   r;   ZwarnsFutureWarningrz   r{   r|   rS   r   r   r   _pq_fastparquet   s   


$r   c                 C  sR   t d}t d}|j|dtdiddt|  |tj	
t| dS )NrD   rE   rI   rF   r&   rK   *)r7   r8   rQ   rP   r;   Zto_csvr{   Zread_csvospathjoin)rT   rU   rV   r   r   r   	_read_csv   s   

r   z#8480)reasonzop,libzpandas.znumpy.optimize_graphTFc              	     s   t dt gid`\}}t|d |d=}|j| ||d |dd }|dd }	||	 }
t fd	d
|	D r>J t fdd
|
D rKJ W d    n1 sUw   Y  W d    d S W d    d S 1 smw   Y  d S )Nplugins)Zscheduler_kwargsaddress)r   )r   c                   S  s
   t tjS r   )r   r   r   r   r   r   r   rk      s   
 z>test_scheduler_highlevel_graph_unpack_import.<locals>.<lambda>c                 S  s   | j tj jS r   )r   r   r"   r   )Zdask_schedulerr   r   r   rk      s    c                 3      | ]}|  V  qd S r   r   r+   modulelibr   r   r/          z?test_scheduler_highlevel_graph_unpack_import.<locals>.<genexpr>c                 3  r   r   r   r   r   r   r   r/      r   )r   r   r   ZcomputeZrun_on_schedulerany)opr   r   r   rT   r   workerscZend_modulesr   Znew_modulesr   r   r   ,test_scheduler_highlevel_graph_unpack_import   s    "r   c                 C  s   | j dddS )Nrj   rM   rN   )rR   ddfr   r   r   _shuffle_op   s   r   c                 C  s   |  dddiS )Nr"   rj   rY   )groupbyZaggr   r   r   r   _groupby_op   s   r   r   c                 C  sp   t d t d}| rt d t d}| |jdd }|j}|| }|	 |	 ks6J d S )NrD   rE   not supporteddask.datasets
2000-01-15end)
r7   r8   _dask_expr_enabledskip
timeseriescountdaskcull__dask_keys__get_all_dependencies)r   rV   datasetsresultgraphculled_graphr   r   r   $test_dataframe_cull_key_dependencies   s   



r   c                    s:  t d t d} t d}| rt d | jdd}d d}i }t|jD ]}d	d
 |j|ff|||f< dd
 ||ff| |f< q)tj	 ||gd}|j
| |j|j}|j} fdd| D }||}	|	j }
|	 }|
|ks{J | }t| D ]\ }|dkr| |f q||ksJ d S )NrD   r   rE   r   r   r   Zcustom_graph_testZcustom_graph_test_0c                 S  rh   r   r   ri   r   r   r   rk     rl   zCtest_dataframe_cull_key_dependencies_materialized.<locals>.<lambda>c                 S  rh   r   r   ri   r   r   r   rk     rl   )dependenciesc                   s   g | ]
}| d fkr|qS )r   r   )r+   kr"   r   r   
<listcomp>  s    zEtest_dataframe_cull_key_dependencies_materialized.<locals>.<listcomp>r   )r7   r8   r   r   r   r;   rL   _namer	   Zfrom_collectionscoreZnew_dd_object_metaZ	divisionsr   r   r   Zkey_dependenciescopyr   listkeysr   )r   rV   r   Zname_0Zdskr1   r   r   Zculled_keysr   Zcached_depsdepsZdeps0r   r   r   1test_dataframe_cull_key_dependencies_materialized   s4   





r   )(
__future__r   r   r7   r8   r   operatorr   r   r   r   Zdistributed.utils_testr   r   Zdask.highlevelgraphr	   Zdask.layersr
   r   r   r   r@   rC   rX   rZ   ra   rg   rq   rx   r}   r   r   markZxfailZparametrizer   r   r   r   r   r   r   r   r   <module>   sT    
		
