
    >ie                    `
   U d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZ d dlZd dlZd dlmZmZmZmZmZmZmZ d dlZd dlmZ d d	l m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+ d d
l,m-Z- d dl.m/Z/ d dl0m1Z1 d dl2m3Z3 d dl4m5Z5m6Z6m7Z7 d dl8m9Z9m:Z:m;Z;m<Z< e=dfe=dfe=dfdZ>de?d<    e@ e=d                    dz  ZA e!e>dd          ZBd ZCd ZDd ZEd ZFd ZGd ZHd ZId ZJd ZKd ZLd  ZMd! ZNejO        P                    d"eQeRg          d#             ZSd$ ZTd% ZUd& ZVd' ZWd( ZXd) ZYd* ZZd+ Z[d, Z\d- Z]ejO        P                    d.d/d0g          d1             Z^d2 Z_d3 Z`d4 Zad5 Zbd6 ZcejO        P                    d.g d7          d8             ZdejO        P                    d.d/d9g          d:             Zed; Zfd< ZgejO        P                    d=eejh        d> g          d?             Zid@ ZjdA ZkdB ZldC ZmdD ZndE ZodF ZpdG ZqdH ZrdI ZsdJ ZtdK ZudL ZvdM ZwdN ZxdO ZydP ZzdQ Z{dR Z|dS Z}ejO        j~        ejO        j        ejO                            dTU          dV                                     ZdW ZdX ZdY ZdZ ZejO                            d[U          ejO        j~        ejO        j        d\                                     Zd] Zd^ Zd_ Zd` Zda Zdb Zdc Zdd Zde Zdf Zdg Zdh Zdi Zdj Zdkefdle	fdmefgZejO        P                    dne          do             Zdp Zdq Zdr Zds Zdt Zdu Zdv Zdw Zdx Zdy Z G dz d{ej!                  Zd| Zd} Zd~ Zd Zd Zd Zd Zd ZejO        P                    d.g d          dd            ZejO        P                    dg d          ejO        P                    dg d          d                         ZejO        P                    dg d          d             Zd Zd Zd Zd Zd Zd Zd ZejO        P                    dg d          d             Zd Zd Zd ZejO        P                    d.g d          d             Z G d de          Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd ZejO        P                    dddg          d             Zd ZejO        P                    dg d          d             ZejO        P                    dg d          d             Zd ZdS )    )annotationsN)BZ2File)Iterator)ProcessPoolExecutor)GzipFile)repeat)groupbyidentityjoinmergepluckuniquevalmap)Bagcollectfrom_delayedinline_singleton_listslazifylazify_taskoptimize	partitionreducebyreifytotal_mem_usage	assert_eq)	Blockwise)Delayed)Graph)	filetextstmpdirtmpfile)add	hlg_layerhlg_layer_topologicalinc   )xr   r)      )r)      r   dsk   r)   c                    | dz  dk    S Nr,   r    r)   s    7lib/python3.11/site-packages/dask/bag/tests/test_bag.pyisevenr4   0       q5A:    c                    | dz  dk    S )Nr,   r+   r1   r2   s    r3   isoddr8   4   r5   r6   c                 N    t           j        dk    sJ t           j        dk    sJ d S )Nr)   r.   )bnamenpartitionsr1   r6   r3   test_Bagr=   8   s*    6S=====Ar6   c                     t                                           t          t                                                    k    sJ d S N)r:   __dask_keys__sortedr-   keysr1   r6   r3   	test_keysrC   =   s3    ??sxxzz 2 2222222r6   c                     t                               t                                                    } | dddgdz  fdg ddz  fgk    sJ d S )NFr+   r.   Tr   r,      )r:   r	   r4   computeresults    r3   test_bag_groupby_pure_hashrJ   A   sU    YYv&&((Fuq!fqj)D)))a-+@AAAAAAAr6   c                     t                               d                                           } t          |           dk    sJ dddgdz  f| v sJ dg ddz  f| v sJ d S )Nc                (    t          |           rdndS )Nevenodd)r4   r2   s    r3   <lambda>z.test_bag_groupby_normal_hash.<locals>.<lambda>I   s    6!99!?% r6   r,   rN   r+   r.   rM   rE   )r:   r	   rG   lenrH   s    r3   test_bag_groupby_normal_hashrQ   G   s|    YY??@@HHJJFv;;!Aq6A:&((((IIIM"f,,,,,,r6   c            
      
  	 t          j        t          d          d          } t          j        t          dd          d          }|                                 }|                                }ddt	          t          j        |           t          t          |                               t	          t          j        | 	          t          t          |                               t	          t          j        | |          t          t          ||                               t	          t          j        | d          fd
|D                        t	          t          j        d|           fd|D                        fdt          ||          D             }t	          t          j        | |d          |           fdt          ||          D             }t	          t          j        | d          |           t          |          		fd|D             }t	          t          j        |                                 |d          |           	fd|D             }t	          t          j        ||                                 d          |           	fd|D             }t	          t          j        d|                                 |          |           t          j
        d          }t	          t          j        | |          fd|D                        t	          t          j        | |          fd|D                        t          j        t          d          d          }t          j        t                    5  t          j        | |           d d d            n# 1 swxY w Y   t          j        t                    5  t          j        |                                 dd           d d d            n# 1 swxY w Y   t          j        t          d          d          }t          j        t                    5  t          j        | ||                                           d d d            n# 1 swxY w Y   t          j        t                    5  t          j        | ||                                           d d d            d S # 1 swxY w Y   d S )Nd   
   r<      r+   r,   r.   c                    | |z   |z   S r?   r1   ar:   cs      r3   myaddztest_bag_map.<locals>.myaddU       1uqyr6   rY   c                (    g | ]} |d           S rT   r1   .0ir[   s     r3   
<listcomp>z test_bag_map.<locals>.<listcomp>[   s#    $=$=$=aUU1b\\$=$=$=r6   r:   c                *    g | ]} d |          S rT   rd   r1   r`   s     r3   rc   z test_bag_map.<locals>.<listcomp>\   s%    &A&A&A!uuR1~~~&A&A&Ar6   c                2    g | ]\  }} ||d           S rS   r:   rZ   r1   ra   rb   jr[   s      r3   rc   z test_bag_map.<locals>.<listcomp>^   s.    
9
9
9FQ55a3
9
9
9r6   ri   c                0    g | ]\  }} |d           S )rS   rZ   r1   rj   s      r3   rc   z test_bag_map.<locals>.<listcomp>a   s)    
4
4
4v155c???
4
4
4r6   rm   c                ,    g | ]} |d           S rh   r1   ra   rb   r[   x_sums     r3   rc   z test_bag_map.<locals>.<listcomp>e   s*    
0
0
055!s###
0
0
0r6   c                ,    g | ]} |d           S rh   r1   ro   s     r3   rc   z test_bag_map.<locals>.<listcomp>h   s*    
0
0
055es###
0
0
0r6   c                ,    g | ]} d |          S )rS   rX   r1   ro   s     r3   rc   z test_bag_map.<locals>.<listcomp>k   s*    
2
2
2!553%1%%%
2
2
2r6   rX   c                (    g | ]} |d           S r_   r1   r`   s     r3   rc   z test_bag_map.<locals>.<listcomp>o   s#    #<#<#<QEE!RLL#<#<#<r6   c                *    g | ]} |d           S rf   r1   r`   s     r3   rc   z test_bag_map.<locals>.<listcomp>p   s%    %@%@%@eeAnnn%@%@%@r6   r'   n   r+   r,   r.   )dbfrom_sequencerangerG   r   maplistzipsumdaskdelayedpytestraises
ValueError)
r:   b2r)   x2solrY   fewer_partsunequalr[   rp   s
           @@r3   test_bag_mapr   O   s   
s444A		%S//r	:	:	:B			A	B    bfUAS]] 3 3444bfUa   $s5!}}"5"5666bfUAr""DUAr):):$;$;<<<bfUAr""$=$=$=$=1$=$=$=>>>bfUB!$$$&A&A&A&Aq&A&A&ABBB
9
9
9
9c!Rjj
9
9
9CbfUAs+++S111
4
4
4
4Q
4
4
4CbfUA%%%s+++FFE
0
0
0
0
0R
0
0
0CbfUAEEGGrS1113777
0
0
0
0
0R
0
0
0CbfUB3///555
2
2
2
2
2r
2
2
2CbfUcQUUWW333S999RAbfUAq!!#<#<#<#<!#<#<#<===bfUA###%@%@%@%@a%@%@%@AAA "5::1===K	z	"	" & &
ua%%%& & & & & & & & & & & & & & & 
z	"	" % %
uaeeggq!$$$% % % % % % % % % % % % % % % uSzzr:::G	z	"	" 2 2
uaB'''//1112 2 2 2 2 2 2 2 2 2 2 2 2 2 2	z	"	" 4 4
ua7b)))113334 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4sH   ,OOO3*P))P-0P-0+R''R+.R++TT
Tc                 N   t          j        t          d          d          } t          j        t          dd          d          }|                                 }|                                }dd|                                                               t          t	          |                    k    sJ |                     |                                          t          t	          ||                    k    sJ |                     d                                          fd|D             k    sJ |                     d	                                          fd
|D             k    sJ |                     |d                                          fdt          ||          D             k    sJ t          |          |                     |                                 d                                          fd|D             k    sJ d S )NrS   rT   rU   rV   r,   r.   c                    | |z   |z   S r?   r1   rX   s      r3   r[   ztest_map_method.<locals>.myadd   r\   r6   c                (    g | ]} |d           S r_   r1   r`   s     r3   rc   z#test_map_method.<locals>.<listcomp>   s#    )B)B)B1%%2,,)B)B)Br6   rd   c                *    g | ]} |d           S rf   r1   r`   s     r3   rc   z#test_map_method.<locals>.<listcomp>   s%    +F+F+FqEE!rNNN+F+F+Fr6   rm   c                0    g | ]\  }} ||d           S r_   r1   rj   s      r3   rc   z#test_map_method.<locals>.<listcomp>   s6     0 0 0"QaB0 0 0r6   c                *    g | ]} |d           S r_   r1   ro   s     r3   rc   z#test_map_method.<locals>.<listcomp>   s'    4T4T4TQUU1eR5H5H4T4T4Tr6   )r,   r.   )rw   rx   ry   rG   rz   r{   r|   r}   )r:   r   r)   r   r[   rp   s       @@r3   test_map_methodr      s*   
s444A		%S//r	:	:	:B			A	B    55<<!!T#eQ--%8%8888855##%%c%B.?.?)@)@@@@@55##%%)B)B)B)B)B)B)BBBBB55"5%%''+F+F+F+FA+F+F+FFFFF55b5!!))++ 0 0 0 0&)!Rjj0 0 0     FFE5525&&..004T4T4T4T4TRS4T4T4TTTTTTTr6   c                    g d} t          j        | d          }dd|                                                              fd| D             k    sJ |                    d                                          fd	| D             k    sJ |                    d
                                          |                                                              fd| D             k    sJ t          j        d          }|                    |                                          fd| D             k    sJ d S )N)r+   r,   )r.   rF   )r'      )      )	   rT   r,   rU   r   c                    | |z   |z   S r?   r1   rX   s      r3   r[   ztest_starmap.<locals>.myadd   r\   r6   c                    g | ]} | S r1   r1   ra   rY   r[   s     r3   rc   z test_starmap.<locals>.<listcomp>   s    )B)B)B%%))B)B)Br6   rT   rm   c                     g | ]
} |d diS rZ   rT   r1   r   s     r3   rc   z test_starmap.<locals>.<listcomp>   s%    /N/N/NAqB/N/N/Nr6   r+   c                D    g | ]} |d                                  iS rm   rG   )ra   rY   
max_secondr[   s     r3   rc   z test_starmap.<locals>.<listcomp>   sA     8 8 8./q)J&&(())8 8 8r6   c                     g | ]
} |d diS r   r1   r   s     r3   rc   z test_starmap.<locals>.<listcomp>   s%    .M.M.M1uua2.M.M.Mr6   r   )rw   rx   starmaprG   r   maxr~   r   )datar:   rZ   r   r[   s      @@r3   test_starmapr      s   444D
1---A    99U##%%)B)B)B)BT)B)B)BBBBB99Ub9!!))++/N/N/N/N/N/N/NNNNN!!J99Uj9))1133 8 8 8 8 8378 8 8     	RA99Ua9  ((**.M.M.M.M.M.M.MMMMMMMr6   c                 .   t                               t                    t          t          fdt          t           j                  D                       } j        | k    sJ j        t                               t                    j        k    sJ d S )Nc                d    i | ],}j         |ft          t          t          t          j         |fff-S r1   )r;   r   filterr4   r:   )ra   rb   rZ   s     r3   
<dictcomp>ztest_filter.<locals>.<dictcomp>   sC     	
 	
 	
 VQK%&&161+!>?	
 	
 	
r6   )	r:   r   r4   r   r-   ry   r<   r~   r;   )expectedrZ   s    @r3   test_filterr      s    	A	
 	
 	
 	
1=))	
 	
 	
 H 6X6QXXf%%*******r6   c                     d } t                               |           }t          |          ddgdz  k    sJ |j        t                               |           j        k    sJ d S )Nc                    | dz  dk    S r0   r1   r2   s    r3   rO   ztest_remove.<locals>.<lambda>   s    !a%1* r6   r+   r.   )r:   remover{   r;   )frZ   s     r3   test_remover      s^    A	A77q!fqj    6QXXa[[%%%%%%%r6   c            	     L   t          t          t                              t          t                    k    sJ t          t          t                              t
                                        t          t          t          dd                    dz            k    sJ d S )Nr+   r   r.   )rA   r{   r:   Lrz   r&   ry   r1   r6   r3   	test_iterr      ss    $q''??fQii''''$quuSzz""##vd5A;;.?.?!.C'D'DDDDDDDr6   funcc                   t          t          j                   | t                    v sJ t          j        d d          | t                    v sJ d | t	          j        t          d                              v sJ d S )Nr'   rx   )strr:   r<   r;   rw   rx   ry   )r   s    r3   	test_reprr      sz    q}a((((6"1":a    dd2#3E!HH#=#=>>>>>>>>r6   c                    ddgddgd} t          | dd          }t          |                    d                    h d	k    sJ t          |                    d
                    h dk    sJ t          |                    d
dg                    h dk    sJ |                    d
dg          j        |                    d
dg          j        k    sJ d S )Nr+   rT   )r,      )r.      )rF   (   )r(   r*   r)   r,   r   >   r+   r,   r.   rF   r+   >   rT   r   r   r   >   rT   r+   r   r,   r   r.   r   rF   )r   setr   r;   dr:   s     r3   
test_pluckr      s    W%'71CDDAAsAAqwwqzz??lll****qwwqzz??.......qww1v#G#G#GGGGG77Aq6??177Aq6??#7777777r6   c                     t          j        g d           t          j        t           fd           t                               dd                     g dk    sJ                      dd           j                             dd           j        k    sJ                      d          j                             dd           j        k    sJ d S )N)Hello Worldc                 H    t                               d                    S Nr   )r{   r   rd   s   r3   rO   z)test_pluck_with_default.<locals>.<lambda>   s    d1771::&6&6 r6   r   )HNW)rw   rx   r   r   
IndexErrorr{   r   r;   rd   s   @r3   test_pluck_with_defaultr      s    
///00A
M*66667774  !!%5%5%55555771d AGGAt$4$4$99999771::?agga..3333333r6   c                    t          j        t          d                                        d           } |                     d          \  }}}t          |          t          t          d                    k    sJ t          |          d t          d          D             k    sJ |j        |                     d          d         j        k    sJ |j        |j        k    sJ d S )NrS   c                    | | dz   | dz   fS )Nr+   r,   r1   r2   s    r3   rO   ztest_unzip.<locals>.<lambda>   s    Aq1ua!e3D r6   r.   c                    g | ]}|d z   S )r,   r1   ra   rb   s     r3   rc   ztest_unzip.<locals>.<listcomp>   s    555Q1q5555r6   r   )rw   rx   ry   rz   unzipr{   r;   )r:   onetwothrees       r3   
test_unzipr      s    
s$$(()D)DEEAggajjOCe99U3ZZ((((((;;55%**55555558qwwqzz!})))))8sxr6   c                 *   t                               t                    } |                                 t	          t
                    k    sJ | j        t                               t                    j        k    sJ t                               t          d          }|j        | j        k    sJ |                                t	          t
                    dt           j        z  z   k    sJ |j        t                               t          d          j        k    sJ t          j	        t          d          d          } d }|                     |t          j        t                                }|                                t          |           k    sJ |j        |                     |t          j        t                                j        k    sJ t          j	        d          }t          |                    d d	                                                    t          d          k    sJ t          j	        d
gdgdggd          }t          |                    t          g                               d                    h dk    sJ d S )NrT   )initialr'   r.   rU   c                X    |                                  } |                     |           | S r?   )copyr#   accr)   s     r3   binopztest_fold.<locals>.binop   s#    hhjj



r6   helloc                0    d                     | |g          S )Nr   )r   rY   r:   s     r3   rO   ztest_fold.<locals>.<lambda>   s    277Aq6?? r6   r   r+   r,   sync	scheduler>   r+   r,   r.   )r:   foldr#   rG   r}   r   keyr<   rw   rx   ry   r   union)rZ   c2r   r   es        r3   	test_foldr      s#   	sA99;;#a&&    5AFF3KKO####	
R	 	 B6QU????::<<3q66B$6666666QVVCV,,00000
qq111A  
 	
uci//A99;;#a&&    5AFF5#)SUUF;;?????
!!Aqvv22Bv??GGIIJJcO O     	1#sQCa888Aqvvc2v&&...@@AAYYYNNNNNNr6   c            
        d } t                               | t          j        t                      t                    }t          |t                    sJ t          |t          t          t          d                                         d S )Nc                0    |                      |           | S r?   )r#   )totr)   s     r3   r   ztest_fold_bag.<locals>.binop  s    



r6   )combiner   out_typer'   )	r:   r   r   r   r   
isinstancer   r{   ry   )r   rZ   s     r3   test_fold_bagr     sr       	
uciEEAaac%((mm$$%%%%%r6   c                 ^   t          t                                                    g dk    sJ t                                          j        t                                          j        k    sJ dt                                          j        v sJ t                                                                                                          dk    sJ t          j        dgdz  d          } |                     d                                                                           g k    sJ d S )Nr   r+   r,   r.   rF   distinctr'   r   2   rU   )	rA   r:   r   r;   countrG   rw   rx   r   bags    r3   test_distinctr     s    !**,,???2222::<<

 11111*****::<<''))Q....

A38
4
4
4C::d$$&&..00B666666r6   c                    d dD             } t          j        | d          }t          t          | d                     }t	          |                    d          |           t	          |                    d           |           d S )	Nc                    g | ]}d |iS r]   r1   r   s     r3   rc   z*test_distinct_with_key.<locals>.<listcomp>  s    
<
<
<C8
<
<
<r6   )
r   r+   r,   r+   r,   r.   r,   r.   rF   r'   r.   rU   c                    | d         S NrY   r1   r2   s    r3   rO   z(test_distinct_with_key.<locals>.<lambda>  s
    af r6   r   rY   c                    | d         S r   r1   r2   s    r3   rO   z(test_distinct_with_key.<locals>.<lambda>  s
    3 r6   )rw   rx   r{   r   r   r   )seqr   r   s      r3   test_distinct_with_keyr     s    
<
<;
<
<
<C

3A
.
.
.CF3$4$455566Hcllsl##X...cll//l00(;;;;;r6   c                 d   t                                           } t          |           ddddddk    sJ t                               d          }t          |          ddddddk    sJ | j        t                                           j        k    sJ | j        |j        k    sJ |j        t                               d          j        k    sJ t	          j        t          d          d          }|                    d           }|                                }t          |          t          t          t          d          d	gdz                      k    sJ t	          j        g d
d          }|                    d                               d          }t          |g            d S )Nr.   r   r,   split_everyr   partition_sizec                    | dk     S NrT   r1   r2   s    r3   rO   z"test_frequencies.<locals>.<lambda>&  s
    QV r6   rT   r+   r   r   r   r   rF   rU   )
r:   frequenciesdictr;   rw   rx   ry   r   r|   r   )rZ   r   r   r   r   bag2s         r3   test_frequenciesr    s   	A77!aA!444444	
1	%	%B881qQ15555556Q]]__)))))6RW7ammm2277777		%))A	6	6	6B	##	$	$B
A77d3uRyy1#(33444444

<<<Q
7
7
7C::d''A'66DdBr6   c                     t          j        g d          } t          |                     d                                                    g dk    sJ d S )N)rY   r:   r:   r:   rZ   rZ   T)sort))r:   r.   )rZ   r,   rY   r+   )rw   rx   r{   r  rG   rd   s    r3   test_frequencies_sortedr  .  sY    
77788A4((0022337U7U7UUUUUUUr6   c                    t          t                              d                    g dk    sJ t                              dd           } t          |           g dk    sJ t                              dd d          }t          |          g dk    sJ | j        |j        k    sJ t                              d          j        t                              d          j        k    sJ d S )	NrF   )rF   rF   rF   r.   c                    |  S r?   r1   r2   s    r3   rO   ztest_topk.<locals>.<lambda>5  s     r6   r   r   r   r   r+   c                    |  S r?   r1   r2   s    r3   rO   ztest_topk.<locals>.<lambda>7  s    ! r6   r,   )r   r  )r{   r:   topkr;   )rZ   r   s     r3   	test_topkr  3  s    q		??lll****	qll##A77lll""""	
||	3	3B88|||####6RW66!99>QVVAYY^++++++r6   r<   r+   r,   c                ^   t          j        g d|           }t          |                    dd                    ddgk    sJ t          |                    dd                    d	dgk    sJ |                    dd          j        |                    dd          j        k    sJ d S )
Nr   r,   r   r.   r   rU   r,   r+   r   r   r  r   r  )rw   rx   r{   r  r;   r<   r:   s     r3   test_topk_with_non_callable_keyr  =  s    
222LLLAqa  !!gv%66666qa  !!ff%5555566!6 AFF1!F$4$4$9999999r6   c                     t          j        g dd          } t          |                     dd                     ddgk    sJ d S )Nr  r,   rU   c                    |S r?   r1   r   s     r3   rO   z0test_topk_with_multiarg_lambda.<locals>.<lambda>G  s    1 r6   r   r   r  )rw   rx   r{   r  rd   s    r3   test_topk_with_multiarg_lambdar  E  sT    
222BBBAqnn--..7F2CCCCCCCr6   c                     t          t                              d                     t          t                              t                              k    sJ d S )Nc                    | dz   S Nr+   r1   r2   s    r3   rO   ztest_lambdas.<locals>.<lambda>K  s
    A r6   )r{   r:   rz   r&   r1   r6   r3   test_lambdasr"  J  sA    oo&&''4c

+;+;;;;;;;r6   c                 "   t          t                                                    dk    sJ t          t                                                    dk    sJ t          t                                                    dk    sJ t          t                                                    dk    sJ t                                                                          du sJ t                                                                          du sJ t                                          j	        t                                          j	        k    sJ t                                          j	        t                                          j	        k    sJ d S )N   r   rF   r   TF)
intr:   r   r}   r   minanyrG   allr   r1   r6   r3   test_reductionsr)  N  s   qwwyy>>Rquuww<<2quuww<<1quuww<<15577??$$$$5577??%%%%5577;!%%''+%%%%5577;!%%''+%%%%%%r6   c                    t                                           j                            d          sJ t                               t          t                    j                            d          sJ t          d t                               t          t                    j        D                       sJ t                               t          t          d          j                            d          sJ d S )Nr}   c              3  j   K   | ].}t          |t                    o|                    d           V  /dS )r   N)r   r   
startswith)ra   ks     r3   	<genexpr>z'test_reduction_names.<locals>.<genexpr>\  sN        78
1c2q||E22     r6   foo)r;   )r:   r}   r;   r,  	reductionr'  r   r~   r1   r6   r3   test_reduction_namesr1  Y  s    5577<""5)))));;sC  %0077777  <=KKS<Q<Q<V        ;;sCe;,,1<<UCCCCCCCr6   c                    t          j        t          d                    } |                     t          t          d          }|                     t          t          d          }|                     t          t          d          }|                                |                                cxk    r|                                k    sn J t          |j                  t          |j                  k    sJ |                     d          }|                     d          }|                                |                                k    sJ t          |j                  t          |j                  k    sJ |j        |j        k    sJ |j        |                     d          j        k    sJ |j        |                                 j        k    sJ d S )N   r,   r  r   r'   )	rw   rx   ry   r0  r}   rG   rP   r~   r   )r:   rZ   r   r   s       r3   test_tree_reductionsr4  b  s{   
r##A	C!,,A	C!,,A	C!,,A99;;!))++4444444444qv;;QV$$$$	!A	!A99;;!))++%%%%qv;;QV$$$$5AE>>>>5AEEaE((,,,,,5AEEGGKr6   )r+   r.   rF   c                   t          t          d                    }t          j        d|           }t          |                                t          |          t          |          z             t          |                                t          |                     t          |                                t          |                     d S )Nr$  rU   )r{   ry   rw   r   meanr}   rP   r   )r<   r   r:   s      r3   test_aggregationr7  w  s    U2YYA
---AaffhhAQ(((aeeggs1vvaggiiQ     r6   rT   c                f   t          j        d          }t          t          d                    }t	          j        ||           }t          |                                |                    |                     t          |                                |                    |                     d S )NnumpyrS   rU   )	r   importorskipr{   ry   rw   rx   r   r6  std)r<   npr   rZ   s       r3   test_non_splittable_reductionsr=    s    		W	%	%Bc

D
;777Aaffhh&&&aeeggrvvd||$$$$$r6   c                     t          t                                          t          j        d                     t          t                                                    t          j        d          k    sJ d S Ng       @)r   r:   r;  mathsqrtfloatr1   r6   r3   test_stdrC    sO    aeeggty~~&&&>>TYs^^++++++r6   c                     t          t                                          d           t          t                                                    dk    sJ d S r?  )r   r:   varrB  r1   r6   r3   test_varrF    s?    aeeggs>>S      r6   	transformc                .    t          j        | d          S )Nr+   rU   )rw   rx   r2   s    r3   rO   rO     s    B4DQTU4V4V4V r6   c                D    | g d          }t                               |t          t                    }t	          |t          t          t          g dt          t          t                                                    t	          t                               |t                    t          t          t          g dt          t          t                                                    |j        t                               |t          t                    j        k    sJ d S )Nrv   )on_selfon_other)r:   r   r8   r4   r   r{   r;   )rG  otherrZ   s      r3   	test_joinrM    s     Iiii  E	uef55Aad6999eT!WW==>>???affUE""DeYYYtAww)O)O$P$PQQQ6QVVE56VBBGGGGGGGr6   c            	        t                               t          t          dt          d          } t          t          t          t           j        dfdft          | j                                                  v sJ t          |           t          t	          t          d t          d                                                    k    sJ | j        t                               t          t          dt          d          j        k    sJ t                               t          d           } t          |           t          t	          t          d t          d                                                    k    sJ d S )Nr   c                    | |z   S r?   r1   r   s     r3   rO   ztest_foldby.<locals>.<lambda>  
    q r6   c                    | |z   S r?   r1   r   s     r3   rO   ztest_foldby.<locals>.<lambda>  s
    a r6   c                    | |z   S r?   r1   r   s     r3   rO   ztest_foldby.<locals>.<lambda>  rP  r6   )r:   foldbyr4   r#   r   r;   r{   r~   valuesr   r   itemsrm   s    r3   test_foldbyrV    s   	aa((AfcAFA;2d16==??6K6KKKKKq66S&*@*@!QGGMMOOPPPPPP6QXXfc1c155:::::	//00Aq66S&*@*@!QGGMMOOPPPPPPPPr6   c            
       
 t                      } dD ]}t          j        t          d          |          }|                    t
          t                    }|                     |g           dD ]}|                    t
          t          |          }|                    t
          t          d|          }|                    t
          t          dt          |          }|                    t
          t          dt          d|          }|                     ||||g           t          j	        |           }	|	d         
t          
fd|	D                       sJ d S )N)r+   r       rS   rU   )FNr,   r.   r  r   c                    g | ]}|k    	S r1   r1   )ra   rfirsts     r3   rc   z.test_foldby_tree_reduction.<locals>.<listcomp>  s    ,,,qU
,,,r6   )r{   rw   rx   ry   rS  r4   r#   extendr~   rG   r(  )r-   nr:   rZ   mr   r   r   gresultsr[  s             @r3   test_foldby_tree_reductionra    s<   
&&C 	% 	%U3ZZQ777HHVS!!

A3$ 	% 	%A!44AaQ77Aa!<<AaaQ??AJJ1a|$$$$	% l3GAJE,,,,G,,,-------r6   c                    t          t                              t                              g dk    sJ t                              t                    j        t                              t                    j        k    sJ t                              d           j        t                              t                    j        k    sJ d S )N)r'   r'   r'   c                &    t          |           dz   S r!  )rP   r]   s    r3   rO   z%test_map_partitions.<locals>.<lambda>  s    c!ffqj r6   )r{   r:   map_partitionsrP   r;   r1   r6   r3   test_map_partitionsre    s      %%&&)))3333C  %)9)9#)>)>)CCCCC00116!:J:J3:O:O:TTTTTTTr6   c            	        d t          d          D             } d t          d          D             }t          j        | d          }t          j        |d          }d	d} || d          }t          t          j        ||d          |           t          |                    |d          |           t          |                    |d          |            || |          }t          t          j        |||          |           t          |                    ||          |           t          |                    ||          |           |                                                    t                    } || t          t          |          t          |          z                      }t          |                    ||          |           t          |                    ||          |           t          j        |          }t          |                    ||          |           t          |                    ||          |           d S )
Nc                8    g | ]}t          j        d d          S irS   randomrandintr   s     r3   rc   z3test_map_partitions_args_kwargs.<locals>.<listcomp>  $    777qc	"	"777r6   rS   c                8    g | ]}t          j        d d          S rh  ri  r   s     r3   rc   z3test_map_partitions_args_kwargs.<locals>.<listcomp>  rl  r6   rT   rU   r   c                    t          |t                    rt          |          n|}d t          | |          D             S )Nc                4    g | ]\  }}t          ||          S r1   )r   )ra   rY   r:   s      r3   rc   zDtest_map_partitions_args_kwargs.<locals>.maximum.<locals>.<listcomp>  s$    222fq!Aq		222r6   )r   r%  r   r|   )r)   ys     r3   maximumz0test_map_partitions_args_kwargs.<locals>.maximum  s<    #As++2F1III22Aq		2222r6   )rp  r   )ry   rw   rx   r   rd  r6  applyr%  r}   rP   r~   r   )r)   rp  dxdyrq  r   dy_means          r3   test_map_partitions_args_kwargsrv    s;   77E#JJ777A77E#JJ777A		!	,	,	,B		!	,	,	,B3 3 3 3 '!r


Cbr222C888b2..444b,,c222
'!Q--CbR00#666b2..444b,,c222ggiiooc""G
'!SQ#a&&))
*
*Cb733S999b113777l7##Gb733S999b11377777r6   c                     t          t                              d d          j        d          } | sJ t	          | t
                    sJ d S )Nc                    | S r?   r1   r2   s    r3   rO   z/test_map_partitions_blockwise.<locals>.<lambda>  s    1 r6   ztest-string)token)r$   r:   rd  r~   r   r   layers    r3   test_map_partitions_blockwiser|    sW     	M::? E LLLeY'''''''r6   c            	         t          j        t          d          d          } dt          t	          |                     dd                              cxk     rdk     sn J dS )	zH
    Number of randomly sampled elements are in the expected range.
      r'   rU   rT   g?*   i,  N)rw   rx   ry   rP   r{   random_sampler]   s    r3   test_random_sample_sizer    sj     	t!444A Db1122339999c99999999r6   c                 f   t          j        t          d          d          } t          j        t
                    5  |                     d           ddd           n# 1 swxY w Y   t          j        t
                    5  |                     d           ddd           dS # 1 swxY w Y   dS )zN
    Specifying probabilities outside the range [0, 1] raises ValueError.
    r   r'   rU   Ng?)rw   rx   ry   r   r   r   r  r]   s    r3   test_random_sample_prob_ranger    s    	r222A	z	"	"  	              	z	"	"  	                 s#   AA#&A#B&&B*-B*c                     t          j        t          d          d          } |                     d          }t	          |          t	          |          k    sJ dS )zf
    Repeated computation of a defined random sampling operation
    generates identical results.
    r   r'   rU   g?Nrw   rx   ry   r  r{   r   s     r3   'test_random_sample_repeated_computationr    sS    
 	r222A	A77d1ggr6   c                 L   t          j        t          d          d          } t          |                     d                    t          |                     d                    k    sJ |                     d          j        |                     d          j        k    sJ dS )z
    Repeatedly defining a random sampling operation yields different results
    upon computation if no random seed is specified.
    r   r'   rU         ?N)rw   rx   ry   r{   r  r;   r]   s    r3   (test_random_sample_different_definitionsr    s    
 	r222A$$%%aooc.B.B)C)CCCCC??3$(<(<(AAAAAAAr6   c                     t          j        t          d          d          } |                     dd          }|                     dd          }t	          |          t	          |          k    sJ dS )zF
    Sampling with fixed random seed generates identical results.
    r   r'   rU   r  i  Nr  rX   s      r3   test_random_sample_random_stater    sh     	r222A	T""A	T""A77d1ggr6   c                    t           t          t          t          g dfff} t	          |           t           t          t          g dffk    sJ t          t          t          g dff} t	          |           | k    sJ t          t          t          t          t
          t          dffff}t          t          t          t
          t          dfff}t	          |          |k    sJ d S )Nrv   rp  )r}   r   rz   r&   r   r   r4   )taskrY   r:   s      r3   test_lazify_taskr    s    %#sIII./0DtsC&; <<<<<Ciii()Dt$$$$	cEFFC#89:;A	cFFC012Aq>>Qr6   c                    | S r?   r1   r2   s    r3   rO   rO   (  s    a r6   c                    t           t          t          t           t          t          dfffft
          dft
          dfd} t           t          t          t          t          dffft
          dft
          dfd}t          |           |k    sJ d S )Nrp  r)   )r)   rY   r:   )r   rz   r&   r   r4   r   r   r   s     r3   test_lazifyr  +  s{    c3(= >?@XX	 	A
 sC&&#!6783xqRUhWWA!99>>>>>>r6   c                    t           dft          ddfd} dt          t           dfdfi}t          | dg          |k    sJ dt          ddfi}t          | dgd          |k    sJ t          | ddg          | k    sJ t          | ddgd          | k    sJ t           dft          ddft          ddfd	} t          | dd
g          | k    sJ dt          dfd} t          | dg          | k    sJ d S )NrY   r:   r+   ri   rZ   Frename_fused_keysr,   r:   rZ   r   r   )rF   r'   )r{   r   r   r   )inpouts     r3   test_inline_singleton_listsr  5  s6   s1c1+
.
.CT3K#
$C!#u--4444C
CC#%888C???? "#Sz22c9999C#su===DDDDs1c1+QQK
@
@C!#Sz22c9999 aX
&
&C!#u--444444r6   c                    t           dft          ddfd} t          | dgd          }|                                dhk    sJ |d         dd          dk    sJ t          j                            d	di          5  t          | dg          |k    sJ 	 d d d            n# 1 swxY w Y   t          | dg          |k    sJ d S )
NrY   r:   r+   ri   rZ   Fr  r  zoptimization.fuse.rename-keys)r{   r   r   rB   r~   configr   )r  outps     r3   test_rename_fused_keys_bagr  I  s   s1c1+
.
.CC#%888D99;;3%9QRR=H$$$$	95A	B	B , ,cU##t+++++, , , , , , , , , , , , , , , C#4''''''s   :BB#&B#c                    t          t                              d                    ddgk    sJ t                              d          dk    sJ t          t                              dd          t                    sJ d S )Nr,   r   r+   )r   r+   Fr   )r{   r:   taker   r   r1   r6   r3   	test_taker  W  sn    q		??q!f$$$$66!99affQf..4444444r6   c                    t          t                              dd                    g dk    sJ t                              dd          dk    sJ t                              dd          dk    sJ t          j        t
                    5  t                              dd	           d d d            d S # 1 swxY w Y   d S )
Nr   r,   rU   )r   r+   r,   r.   rF   r   r  r.   )r   r+   r,   r+   r'   )r{   r:   r  r   r   r   r1   r6   r3   test_take_npartitionsr  ]  s    qa(())-?-?-?????66!6$$(:::::66!6$$	1111	z	"	" ! !	qa   ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! !s   B99B= B=c                 x   t           j                            d          5  t          j        t
                    5  t                              d           d d d            n# 1 swxY w Y   t          j        t
                    5  t                              d           d d d            n# 1 swxY w Y   t          j	        d          5 } t                              dd           t                              dd	
           d d d            n# 1 swxY w Y   | rJ 	 d d d            d S # 1 swxY w Y   d S )Nr   r   rS   r   Trecordr,   rU   F)warn)
r~   r  r   r   warnsUserWarningr:   r  warningscatch_warningsr  s    r3   test_take_npartitions_warnr  e  s    
6	*	* 
 
\+&& 	 	FF3KKK	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 \+&& 	 	FF1III	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 $D111 	"VFF1!F$$$FF15F!!!	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
s|   D/A"D/"A&	&D/)A&	*D/B-!D/-B1	1D/4B1	5D/9DD/D	D/D	D//D36D3c                 V    t          t          d g d          t                    sJ d S )Nc                    | S r?   r1   r2   s    r3   rO   z"test_map_is_lazy.<locals>.<lambda>v  s    A r6   rv   )r   rz   r   r1   r6   r3   test_map_is_lazyr  u  s.    c++yyy118<<<<<<<r6   c                 |    t          t          t                                                    t                    sJ d S r?   )r   r	  r:   r  r1   r6   r3   "test_can_use_dict_to_make_concreter  y  s.    d1==??++T2222222r6   Hangs)reasonc                     t          j        ddg          } | j        dk    sJ t          j        d          }|j        dk    sJ d|                    d          v sJ d S )Nzhttp://google.comzhttp://github.comr,   z:http://raw.githubusercontent.com/dask/dask/main/README.rstr+   s   Dask
rT   )rw   from_urlr<   r  r   s     r3   test_from_urlr  }  sq     	(*=>??A=A
PQQA=Ar

""""""r6   c                 V   t          ddd          5 } d t          j        |           D             t          d          k    sJ d t          j        d          D             t          d          k    sJ 	 d d d            n# 1 swxY w Y   t	          j        t          d            d S )	NzA
BzC
D)za1.logza2.logc                6    h | ]}|                                 S r1   stripra   lines     r3   	<setcomp>z!test_read_text.<locals>.<setcomp>  s     ;;;

;;;r6   ABCDc                6    h | ]}|                                 S r1   r  r  s     r3   r  z!test_read_text.<locals>.<setcomp>  s     @@@

@@@r6   za*.logc                 *    t          j        d          S )Nznon-existent-*-path)rw   	read_textr1   r6   r3   rO   z test_read_text.<locals>.<lambda>  s    bl3H&I&I r6   )r    rw   r  r   r   r   r   )fnss    r3   test_read_textr    s    	f77	8	8 PC;;c):):;;;s6{{JJJJ@@h)?)?@@@CKKOOOOOP P P P P P P P P P P P P P P M*IIJJJJJs   A#BB
Bc            	        t                      5 } t          | d          5 }|                    dt          j        z                                   dz             d d d            n# 1 swxY w Y   t          j        | d          }t          j        |           }t          |j	                  dk    sJ t          t          t          |j                                                            t          t          t          |j                                                            k    sJ t          j        | gd          }t          |          t          |          k    sJ 	 d d d            d S # 1 swxY w Y   d S )NwbzHello, world!rS   	blocksizer'   )r"   openwriteoslinesepencoderw   r  rP   r~   r{   rz   r   r  )fnr   r:   rZ   r   s        r3   test_read_text_larger    s   	 	"b"d^^ 	CqGG_rz199;;cABBB	C 	C 	C 	C 	C 	C 	C 	C 	C 	C 	C 	C 	C 	C 	CLs+++L16{{QCQU[[]]++,,Saekkmm5L5L0M0MMMMML"---Aww$q''!!!!!	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	"s5   E08A$E0$A(	(E0+A(	,C6E00E47E4c                    t                      5 } t          | d          5 }|                    dt          j        z                       d          dz             d d d            n# 1 swxY w Y   t          j        | dd          }t          j        | d          }t          |j	                  dk    sJ |j
                                                            d           }|j
                                                            d	           }t          |          t          |          k    sJ t          j        | gdd          }t          |          t          |          k    sJ 	 d d d            d S # 1 swxY w Y   d S )
Nr  u	   你好！gb18030rS   )r  encodingr  r'   c                ,    |                      d          S Nutf-8r  r2   s    r3   rO   z)test_read_text_encoding.<locals>.<lambda>      AHHW,=,= r6   c                ,    |                      d          S r  r  r2   s    r3   rO   z)test_read_text_encoding.<locals>.<lambda>  r  r6   )r"   r  r  r  r  r  rw   r  rP   r~   r   r  rz   r{   )r  r   r:   rZ   b_encc_encr   s          r3   test_read_text_encodingr    s   	 "b"d^^ 	HqGG[2:-55i@@3FGGG	H 	H 	H 	H 	H 	H 	H 	H 	H 	H 	H 	H 	H 	H 	HLsY???Li00016{{Q!!"="=>>!!"="=>>E{{d5kk))))L"yAAAAww$q''!!!!!" " " " " " " " " " " " " " " " " "s5   E;9A%E;%A)	)E;,A)	-D E;;E?E?c                 "   t          d          5 } d}t          | d          }|                    |           |                                 t	          j        t                    5  t          j        | dd           d d d            n# 1 swxY w Y   t          j        | d           }|j	        dk    sJ d	
                    |                                          |                                k    sJ 	 d d d            d S # 1 swxY w Y   d S )
Ngzsx  Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
Hello, world!
r  r   
)r  linedelimiterr  r+   r   )r"   r   r  closer   r   r   rw   r  r<   r   rG   decode)r  r   r   rZ   s       r3   test_read_text_large_gzipr    st   	 5"'R					]:&& 	? 	?Lr>>>>	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? Lt,,,}!!!!wwqyy{{##t{{}}444445 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5s7   AD%B	=D	B	DB	A%DDDz(https://github.com/dask/dask/issues/6914c                    t          j        d           d} t          j        dt	          d                    }|                    d          | k    sJ t          j        g dt	          d                    }|j        d	k    sJ d S )
Ns3fs)z(total_bill,tip,sex,smoker,day,time,size
z"16.99,1.01,Female,No,Sun,Dinner,2
z 10.34,1.66,Male,No,Sun,Dinner,3
z21.01,3.5,Male,No,Sun,Dinner,3
z 23.68,3.31,Male,No,Sun,Dinner,2
zs3://tip-data/t*.gzT)anon)storage_optionsr'   )zs3://tip-data/tips.gzzs3://tip-data/tips.jsonzs3://tip-data/tips.csvr.   )r   r:  rw   r  r	  r  r<   )	five_tipsr   rZ   s      r3   test_from_s3r    s     I 	*DdOOOLLLA66!99	!!!! 	VVV$	 	 	A =Ar6   c                     t          j        g dd          } t          | j                  dk    sJ t	          |           h dk    sJ d S )N)r+   r,   r.   rF   r'   r.   rU   >   r+   r,   r.   rF   r'   )rw   rx   rP   r~   r   rd   s    r3   test_from_sequencer    sU    
a888Aqv;;!q66___$$$$$$r6   c                     t          t          d                    } t          j        |           }t	          |          t	          |           k    sJ d S )Ni  )r{   ry   rw   rx   r   )r   r:   s     r3   test_from_long_sequencer    sH    U4[[A
Aq66SVVr6   c                     t          j        d           t          j        g           } | j        dk    sJ |                     ddi                                          }|j        s
J d            d S )Ndask.dataframer+   rY   r%  metazDataFrame is not empty)r   r:  rw   rx   r<   to_dataframerG   empty)r:   dfs     r3   test_from_empty_sequencer    sw    
()))
A=A	
c5\	*	*	2	2	4	4B8--------r6   c                    t                               t                     } | j        t           j        dz  k    sJ t          |           d t          D             k    sJ t          j        g d          }t          j        g d          }|                    |          }t          |          d dD             k    sJ |j        | j        k    sJ |j        |                    |          j        k    sJ d S )Nr,   c                *    h | ]}t           D ]}||fS r1   )r   ra   rb   rk   s      r3   r  ztest_product.<locals>.<setcomp>  s)    444!!44Q1v4444r6   )r+   r,   r.   rF   rT   r   r   c                     h | ]}d D ]}||fS )r  r1   r  s      r3   r  ztest_product.<locals>.<setcomp>  s)    IIILIIqq!fIIIIr6   )r:   productr<   r   r   rw   rx   r;   )r   r)   rp  zs       r3   test_productr    s    	
1B>Q]A-----r7744q4444444
&&A
&&A			!Aq66IIlIIIIIII6RW6QYYq\\&&&&&&&r6   c            	        t          j                    5 } t          t          t	          d          d|            t          |                     d                    ddhk    sJ t          |                     d                    ddhk    sJ t          |                     d                    ddhk    sJ t          t          t          d| d                    ddgfddgfgk    sJ 	 d d d            d S # 1 swxY w Y   d S )	Nr   r.   r   r+   rF   r,   r'   r   )	partdPickler   r
   ry   r   getrA   r   )ps    r3   test_partition_collectr    sF   	 K1(E!HHa+++15588}}A&&&&15588}}A&&&&15588}}A&&&&gh1b1122A3x!aS6JJJJJJK K K K K K K K K K K K K K K K K Ks   CC77C;>C;c                 t   t                               t                    } t          |           }|g dg dg dg dg ddk    sJ | j        t           j        k    sJ | j        t                               t                    j        k    sJ | j        t                               d           j        k    sJ d S )Nr   r   r   r+   r+   r+   r,   r,   r,   r.   r.   r.   rF   rF   rF   r   c                    | dz   S r!  r1   r2   s    r3   rO   ztest_groupby.<locals>.<lambda>  s
    Q r6   )r:   r	   r
   r	  r<   r;   )rZ   rI   s     r3   test_groupbyr	    s    			(A!WWF9999999999      =AM))))6QYYx((-----6QYY//4444444r6   c                     t          j        g dg dg dg          } t          |                     d                    }t	          t
          |          g dg dgg dgdk    sJ d S )Nrv   )r+   rF   r   r,   r.   rF   r   r   )rw   rx   r	  r	   r   rA   r:   rI   s     r3   test_groupby_with_indexerr    sx    
)))YYY			:;;A!))A,,F&&!!)))YYY)?YYYK%P%PPPPPPPr6   c                     t                               d d          } t          |           }|g dg dg dg dg dd	k    sJ | j        dk    sJ d S )
Nc                    | S r?   r1   r2   s    r3   rO   z7test_groupby_with_npartitions_changed.<locals>.<lambda>  s     r6   r+   rU   r  r  r  r  r  r   )r:   r	   r	  r<   )rI   result2s     r3   %test_groupby_with_npartitions_changedr    s    YY{{Y22F6llG9999999999      """"""r6   c                     ddl m}  t          j                            |           5  t
                              d d                                           d d d            d S # 1 swxY w Y   d S )Nr   r  r   c                    | S r?   r1   r2   s    r3   rO   z2test_groupby_with_scheduler_func.<locals>.<lambda>,  s    A r6   r+   rU   )dask.threadedr  r~   r  r   r:   r	   rG   r  s    r3    test_groupby_with_scheduler_funcr  (  s    !!!!!!	3	'	' 8 8			++1	--557778 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8s   0A$$A(+A(c                    t          j        g d          } t          j        g d          }t          j        | |g          }t          |          g dk    sJ |j        t          j        | |g          j        k    sJ d S )Nrv   rF   r'   r   r+   r,   r.   rF   r'   r   )rw   rx   concatr{   r;   rX   s      r3   test_concatr  /  s    
##A
##A
	1a&A77(((((((6RY1v&&+++++++r6   c                     t          j        dgddgg          } t          |                                           g dk    sJ |                                 j        |                                 j        k    sJ d S )Nr+   r,   r.   rv   )rw   rx   r{   flattenr;   rd   s    r3   test_flattenr  7  sn    
1#1v''A					))))99;;qyy{{///////r6   c                     t          j        ddg          } t          j        ddg          }t          j        |                     t                    |g          }t          |          g dk    sJ d S )Nr+   r,   rF   r'   )r,   r.   rF   r'   )rw   rx   r  rz   r&   r{   )rY   r:   rI   s      r3   test_concat_after_mapr   =  sh    
!Q  A
!Q  AYc

A''F<<<<<''''''r6   c                     t                               d           } t          | j         }t	          |           t	          |          k    sJ | j        |j        k    sJ d S )Nc                    | dz   S r!  r1   r2   s    r3   rO   ztest_args.<locals>.<lambda>E  s
    A r6   )r:   rz   r   _argsr{   r<   )rZ   r   s     r3   	test_argsr$  D  sX    	ooAQWA77d1gg=AM))))))r6   c                 P   t          j        d          } t          j        d          }d }dgg ddgd}t          |dd	          }|                    |                                d
dg          }|                                }| j                            ||                    ddd          d           |                    d
dg          }| j                            ||d            |||           |                    ddg          }| j                            ||d            |||           |	                    d           }|                                }| j                            ||d            |||           |j
        |                                j
        k    sJ |ddgfD ]A}|                    |          }| j                            ||d            |||           Bt          j        t                    5  |                    d
dg|           d d d            n# 1 swxY w Y   |                    d           }t          j        t                    5  |                                 d d d            n# 1 swxY w Y   |                    d
          }|d
g         }|                    |          }| j                            ||d            |||           |                    d
t          d          i          }t!          j        t          d          d          }t$          t&          fD ]T}	|                    |	                              |          }| j                            ||d            |||           Ud S )Nr  pandasc                ~    t          fdt          j        |                                  D                       sJ d S )Nc              3  \   K   | ]&}|j         j         k                                    V  'd S r?   )dtypesr(  )ra   r   r   s     r3   r.  z9test_to_dataframe.<locals>.check_parts.<locals>.<genexpr>Q  sI       
 
/0QX#((**
 
 
 
 
 
r6   )r(  r~   rG   
to_delayed)r  r   s    `r3   check_partsz&test_to_dataframe.<locals>.check_partsP  s^     
 
 
 
48L"--//4R
 
 
 
 
 	
 	
 	
 	
 	
r6   r   )rT   r   )rS   rV   ))testr   )r,  r+   )r,  r,   r,  r.   rY   r:   )columnsr   r+   r   F)check_index)rY   i8)r:   r/  r  c                @    t          t          ddg|                     S )NrY   r:   )r	  r|   r2   s    r3   rO   z#test_to_dataframe.<locals>.<lambda>d  s    S#sQ//00 r6   )r-  r  c                    | d         dk    S )NrY   rV   r1   r2   s    r3   rO   z#test_to_dataframe.<locals>.<lambda>u  s    AcFSL r6   rS   r'   rU   )r   r:  r   	DataFramerG   r  utilsr   renamerz   _namer   r   r   r   ry   rw   rx   itertuplerd  )
ddpdr+  r-   r:   r   r  r  r   r   s
             r3   test_to_dataframer:  L  s9   		-	.	.B		X	&	&B
 
 

  r:@V
W
WCCA
,,qyy{{S#J,
7
7C 
		BHr3::AA.>.>:??USSS	
c
	+	+BHr3E222KC	
k;7	8	8BHr3E222KC 	
0011A	
		BHr3E222KC8q~~''----- {K01  ^^^&&
2s666B 
z	"	" 5 5	Sz4445 5 5 5 5 5 5 5 5 5 5 5 5 5 5 
((	)	)B	z	"	"  
               	
A
se*C	
S	!	!BHr3E222KC ,,U3ZZ(
)
)C
s333AE]  a  --3-77
2s666B s$   H99H= H=3JJJr  bz2r   z
ext,myopenc                   t          j        g dd          }t                      5 }|                    t          j                            |d| z             d          }t          j        |ddi t          j        	                    t          j                            |d	| z                       sJ  |t          j                            |d	| z             d
          }|
                                }t          |d          r|                                }d|v sJ |                                 d d d            d S # 1 swxY w Y   d S )Nabc123xyzr,   rU   *.Fr   r   r   1.rbr  r@  )rw   rx   r!   to_textfilesr  pathr   r~   rG   existsreadhasattrr  r  )extmyopenr:   dirrZ   r   texts          r3   test_to_textfilesrM    s`   
...A>>>A	 
SNN27<<TCZ88%NHHa*6***w~~bgll3s
;;<<<<<F27<<TCZ00$77vvxx4"" 	!;;==D}}}}				
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
s   DD??EEc                    g d} t          j        | d          }t                      5 }|                    |           t          j        t
          j                            |d          d                              t                                        t          j
                                                  }| |k    sJ 	 d d d            d S # 1 swxY w Y   d S )NrY   r:   rZ   r   r   r   r_  hrb   rk   r-  lr^  r]  or      rU   *asciir  )rw   rx   r!   rD  r  r  rE  r   rz   r   r  rG   )r   r:   dnr  s       r3   /test_to_textfiles_name_function_preserves_orderrW    s     C$ 	"---A	 	R	r Lb#..AAASXXS^^WYY	 	 czzzzz	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	s   BCCCc                     g d} t          j        | d          }t                      5 }|                    |t                     d d d            d S # 1 swxY w Y   d S )NrO  rS  rU   )name_function)rw   rx   r!   rD  r   )r   rY   rV  s      r3   $test_to_textfiles_name_function_warnrZ    s      C$ 	"---A	 .R	r---. . . . . . . . . . . . . . . . . .s   AAAc            	        t          j        g dd          } t          D ]5\  }}t                      5 }|                     t
          j                            |d|z             dd          }t          j	        |  t
          j        
                    t
          j                            |d|z                       sJ  |t
          j                            |d|z             d	          }|                                }t          |d
          r|                    d          }d|v sJ |                                 d d d            n# 1 swxY w Y   7d S )N)u   汽车u   苹果   天气r,   rU   rA  r  F)r  rG   rB  rC  r  r\  )rw   rx   ext_openr!   rD  r  rE  r   r~   rG   rF  rG  rH  r  r  )r:   rI  rJ  rK  rZ   r   rL  s          r3   test_to_textfiles_encodingr^    s{   
777QGGGA  VXX 	S$*--	5   A L!7>>"',,sD3J"?"?@@@@@rw||C44d;;A6688DtX&& .{{9--t####GGIII	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 s   D
E

E	E	c                 H   t          j        g dd          } t                      5 }t                      5 }|                     ||g           t          j                            |          sJ t          j                            |          sJ 	 d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   t                      5 }|                     |           t          j                            |          sJ t          j                            t          j                            |d                    sJ 	 d d d            n# 1 swxY w Y   t          j
        t                    5  |                     d           d d d            d S # 1 swxY w Y   d S )Nr=  r,   rU   z0.partr'   )rw   rx   r"   rD  r  rE  rF  r!   r   r   r   	TypeError)BrY   r:   dirnames       r3   test_to_textfiles_inputsrc    sb   
...A>>>A	 %aYY 	%!NNAq6"""7>>!$$$$$7>>!$$$$$$	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	%% % % % % % % % % % % % % % % 
 ?W	ww~~g&&&&&w~~bgll7H==>>>>>>? ? ? ? ? ? ? ? ? ? ? ? ? ? ?
 
y	!	!  	q                 sZ   B4ABB4B!	!B4$B!	%B44B8;B8A6EEE4FFFc                 T   t          j        g dd          } t                      5 }dD ]d}|                     |g|           t	          |          5 }|                                }d d d            n# 1 swxY w Y   |dd|rdnd	gk    sJ e	 d d d            d S # 1 swxY w Y   d S )
NrX   r+   rU   )FT)last_endlineza
zb
zc
rZ   )rw   rx   r"   rD  r  	readlines)r:   r  re  r   rI   s        r3   test_to_textfiles_endlinesrg    sW   
a888A	 Lb' 	L 	LLNNB4lN;;;b 'Q' ' ' ' ' ' ' ' ' ' ' ' ' ' 'eU\,JEEsKKKKKK		LL L L L L L L L L L L L L L L L L Ls5   ,BA4(B4A88B;A8<BB!$B!c                      t          j        g dd           dt           j                  v sJ dt           j                  v sJ t	           j                                                  g dk    sJ t	           j                            d                    dd	gd
dgdd	ggk    sJ t	           j                            d                    ddgk    sJ t          j	        t           fd            j                            d          j         j                            d          j        k    sJ  j                            d          j         j                            d          j        k    sJ d S )NAlice Smithz	Bob JonesCharlie Smithr,   rU   splitmatchzalice smithz	bob joneszcharlie smith AliceSmithBobJonesCharliez*Smithrj  rk  c                      j         j        S r?   )r   	sfohsofhfrd   s   r3   rO   z'test_string_namespace.<locals>.<lambda>  s    !%/ r6   z*John)rw   rx   rK  r   r{   lowerrl  rm  r   r   AttributeErrorr;   rd   s   @r3   test_string_namespacery    s|   
FFFTUVVVAc!%jj    c!%jj    "O"O"OOOOOC  !!	'		G&    
 H%%&&=/*JJJJJ
M."9"9"9"9:::5;;x  %X)>)>)CCCCC5;;x  %W)=)=)BBBBBBBr6   c                     t          j        g dd          } t          | j                                                  g dk    sJ d S )Nri  r,   rU   rn  )rw   rx   r{   r   rw  rd   s    r3   "test_string_namespace_with_unicoder{    sO    
FFFTUVVVA"O"O"OOOOOOOr6   c                     t          j        g dd          } t          | j                                                  ddgddgddggk    sJ d S )	Nri  r,   rU   rp  rq  rr  rs  rt  )rw   rx   r{   r   rl  rd   s    r3   test_str_empty_splitr}    so    
FFFTUVVVA	'		G#      r6   c                     t          j        g dg dgd          } d }|                     |          }t          |          g dg dgk    sJ d S )Nrv   r  r,   rU   c              3  "   K   | D ]	}|dz   V  
d S r!  r1   )r   r)   s     r3   r   z*test_map_with_iterator_function.<locals>.f%  s.       	 	Aa%KKKK	 	r6   r  )r'   r   r   )rw   rx   rz   r{   )r:   r   rZ   s      r3   test_map_with_iterator_functionr  "  sn    
)))YYY/Q???A   	
aA77yyy))),,,,,,,r6   c                     t          j        g d          } |                     d                                           }t	          |t
                    rJ d S )Nrv   c                    | dz   S r!  r1   r2   s    r3   rO   z8test_ensure_compute_output_is_concrete.<locals>.<lambda>0  s
    QU r6   )rw   rx   rz   rG   r   r   r  s     r3   &test_ensure_compute_output_is_concreter  .  sQ    
##AUU??##++--F&(+++++++r6   c                      e Zd ZddZd ZdS )
BagOfDictsNc                8    |                      fd          S )Nc                0    |                                S r?   r  )r   defaultr   s    r3   rO   z BagOfDicts.get.<locals>.<lambda>6  s    !%%W"5"5 r6   rz   )selfr   r  s    ``r3   r  zBagOfDicts.get5  s#    xx55555666r6   c                <    fd}|                      |          S )Nc                    | <   | S r?   r1   )r   r   values    r3   setterzBagOfDicts.set.<locals>.setter9  s    AcFHr6   r  )r  r   r  r  s    `` r3   r   zBagOfDicts.set8  s6    	 	 	 	 	 	 xxr6   r?   )__name__
__module____qualname__r  r   r1   r6   r3   r  r  4  s7        7 7 7 7         r6   r  c                    t          t          j        dddiig          j         } |                     d                              d                                          d         dk    sJ |                     d                              dd                                          d         dddk    sJ t          |                     d                              d          t                     sJ d S )NrY   r:   rZ   r   r   zEXTENSIBILITY!!!r:   r   )r  rw   rx   r#  r  rG   r   r   )dictbags    r3   test_bag_class_extendr  @  s    "*S3*,=+>??EFG;;s$$,,..q1S8888;;s%788@@BB1EJ J     gkk#&&**3//<<<<<<<r6   c                    d                                 } t                      5 }t          |d          5 }|                    |            d d d            n# 1 swxY w Y   t	          j        |          }|                                d         |                     d          k    sJ 	 d d d            d S # 1 swxY w Y   d S )Nu   €r  r   r  )r  r"   r  r  rw   r  rG   r  )bin_datar  r   rY   s       r3   
test_gh715r  J  s)     H	 :b"d^^ 	qGGH	 	 	 	 	 	 	 	 	 	 	 	 	 	 	Lyy{{1~!9!999999	: : : : : : : : : : : : : : : : : :s5   B6A
B6A	B6A	A
B66B:=B:c                     t          j        g d                              d           } |                     d           d S )Nrv   c                    | dz   S r!  r1   r]   s    r3   rO   z1test_bag_compute_forward_kwargs.<locals>.<lambda>T  s
    !a% r6   rT   )bogus_keyword)rw   rx   rz   rG   r2   s    r3   test_bag_compute_forward_kwargsr  S  s@    
##''88AIIBIr6   c                    t          j        g dd          } |                     t                                                    \  }} }t          d || |fD                       sJ |                                 ddgk    sJ t          j        g dd          } |                                                                 }t          |t                    sJ |                                dk    sJ d S )Nr  r.   rU   c              3  @   K   | ]}t          |t                    V  d S r?   )r   r   ra   r)   s     r3   r.  z"test_to_delayed.<locals>.<genexpr>[  s,      99!z!W%%999999r6   rF   r'      )
rw   rx   rz   r&   r*  r(  rG   r}   r   r   )r:   rY   rZ   ts       r3   test_to_delayedr  X  s    
+++;;;AeeCjj##%%GAq!991ay9999999999;;1a&    
+++;;;A	Aa!!!!!99;;"r6   c                   t          j        g dd          }|                    t                                        t                                        t                    }|                                \  }t          t          |j                            }|                    d          dk    sJ |	                                |	                                k    sJ |                    d          \  }t          |j                  t          |j                  k    sJ |	                                |	                                k    sJ |
                                |
                                k    sJ |                                }|                                }t          t          |j                            }|	                                |	                                k    sJ |                    d          dk    sJ |                    d          }t          |j                  t          |j                  k    sJ |	                                |	                                k    sJ |
                                |
                                k    sJ |                    t          |           d          \  }t          t          |j                            }|                    d          dk    sJ d S )	Nr  r+   rU   r   Foptimize_graphr   r   )rw   rx   rz   r&   r*  r   r	  r~   r   __dask_layers__rG   r}   rD  )r!   r:   r   r   rL  d2r)   s          r3   test_to_delayed_optimize_graphr  d  sy   
+++;;;A	
s			 	 	%	%B
--//CQtAF||D::g!####""4"4"6"66666===..DR==DMM))))2#5#5#7#7777799;;"**,,&&&&
A	AtAF||D!"3"3"5"55555::g!####	
U	+	+B==DLL((((1#4#4#6#6666699;;"**,,&&&&
//#f++u/
5
5CQtAF||D::g!######r6   c                    ddl m}   | g d           | g d           | g d          }}}t          |||g          }|j        t          |||g          j        k    sJ t	          |t
                    sJ t          |          g dk    sJ   | t                    |          }t          j	                            |          }|
                                |
                                cxk    rdk    sn J d S )Nr   r   rv   r  )r   r   r   )	r+   r,   r.   rF   r'   r   r   r   r   r   )dask.delayedr   r   r;   r   r   r{   r}   rw   ItemrG   )r   rY   r:   rZ   bb
asum_value	asum_items          r3   test_from_delayedr    s#   $$$$$$giii  '')))"4"4ggiii6H6H!qA	q!Qi	 	 B7lAq!9--22222b#882222222aJ$$Z00I9#4#4#6#6;;;;!;;;;;;;;r6   c                    ddl m}  d } | |d          t          j        fdt	          d          D                       }t          j        |                                |                    d                                          |                    d                                                                          d	
          dk    sJ d S )Nr   r  c                4    d t          |           D             S )Nc              3  "   K   | ]
}d ddgiV  dS )
operationsr+   r,   Nr1   )ra   _s     r3   r.  zCtest_from_delayed_iterator.<locals>.lazy_records.<locals>.<genexpr>  s+      9911v&999999r6   ry   )r]  s    r3   lazy_recordsz0test_from_delayed_iterator.<locals>.lazy_records  s    99a9999r6   F)purec                &    g | ]} d           S )r'   r1   )ra   r  delayed_recordss     r3   rc   z.test_from_delayed_iterator.<locals>.<listcomp>  s#    @@@!??1--@@@r6   r'   r  r   r   )   r  r   )	r  r   rw   r   ry   rG   r   r   r  )r   r  r   r  s      @r3   test_from_delayed_iteratorr    s    $$$$$$: : : gl777O
/@@@@uQxx@@@
A
AC:				,%%''		,''))//11	  
 
     r6   c                     dD ]n} t          j        d|           }t          |j                  | k    sJ |j        | k    sJ t          |          t          t          d                    k    sJ od S )Nr+   r   rT      rS   rU   )rw   ry   rP   r~   r<   r{   r  s     r3   
test_ranger    s    % + +HSk22216{{k))))}++++Aww$uSzz*******	+ +r6   r  r~  c                   t          j        t          d|d          |           }t          j        t          d|d          |           }t          j        ||          }|j        |j        k    sJ |j        |j        k    sJ t          |          t          t          t          d|d          t          d|d                              k    sJ d S )Nr   r,   rU   r+   )rw   rx   ry   r|   r<   r{   )r<   hievensoddspairss        r3   test_zipr    s    U1b!__+FFFEE!ROOEEEDF5$E 11111 00000;;$s5B??E!ROODDEEEEEEEEr6   nin)r+   r,   r         nout)r+   r,   r'   r3  r  c                ,   t          j        t          d          |           }|                    |          }|j        |k    sJ t          ||           t          j        |j        |                                          }t          |          sJ d S )NrS   rU   )
rw   rx   ry   repartitionr<   r   r~   r  r@   r(  )r  r  r:   rZ   r`  s        r3   test_repartition_npartitionsr    s     	s555A	$''A=D    aOOOhqvq0011Gw<<r6   z	nin, nout)	r+   r+   )r,   r+   )r'   r+   r   )r,   r,   )r'   r,   )r+   r'   )r,   r'   )r'   r'   c                ,   t          j        t          dd          |           }t          |                    t
                                                              }|                    ||z            }|j        |k    sJ t          ||           d S )Nr+   rS   rU   r  
rw   rx   ry   r}   rd  r   rG   r  r<   r   )r  r  r:   	total_memrZ   s        r3   test_repartition_partition_sizer    s     	q#C888AA$$_55==??@@I	i4&799A=D    aOOOOOr6   c                    t          j        t          dd          d          } t          |                     t
                                                              }|                     |dz            }|j        dk    sJ t          | |           |                    |dz            }|j        dk    sJ t          ||           d S )Nr+   rS   rU   r,   r  r'   r  )r:   r  rZ   r   s       r3   (test_multiple_repartition_partition_sizer    s    
q#A666AA$$_55==??@@I	i1n66A=AaOOO	i1n66A=AaOOOOOr6   c                 p   t          j        d          t          j        fdt	          d          D             d          } t          |                     t                                                              }|dz  }| 	                    |          }|j
        dk    sJ t          | |           d S )Nr9  c                T    g | ]$}                     t          d                     %S )rS   )arrayry   )ra   r  r<  s     r3   rc   zBtest_repartition_partition_size_complex_dtypes.<locals>.<listcomp>  s+    AAA1"((5::..AAAr6   rF   r+   rU   r  )r   r:  rw   rx   ry   r}   rd  r   rG   r  r<   r   )r:   r  new_partition_sizerZ   r<  s       @r3   .test_repartition_partition_size_complex_dtypesr    s    		W	%	%B
AAAAaAAAqQQQAA$$_55==??@@I"a	%788A=AaOOOOOr6   c                 B   t          j        t          d          d          } |                     d          }| j        |j        k    sJ |                     d          }| j        |j        k    sJ |j        |j        k    sJ |                     d          }| |u sJ d S )NrS   r'   rU   r,   r   )rw   rx   ry   r  r;   r  s      r3   test_repartition_namesr    s    
s333A	aA6QV	bA6QV6QV	aA666666r6   c                     t          j        t                    5  t          j        t          d                    } |                     dd           d d d            d S # 1 swxY w Y   d S )NrT   r'   5MiB)r<   r  )r   r   r   rw   rx   ry   r  r   s    r3   test_repartition_input_errorsr    s    	z	"	" > >uRyy))Af===> > > > > > > > > > > > > > > > > >s   9A  A$'A$c                 F   g dddgg ddgg} d t          |           D             }t          j        |dt          |                     }|                    t
                    }|j        |                    t
                    j        k    sJ |j        |                    t
          d          j        k    sJ |                                g d	k    sJ |                    t
          d                                          g d
k    sJ |                    t
                                        t                                                    g dk    sJ t          j
        g dd          }|                    t
                                                    g dk    sJ d S )Nrv   rF   r'   r   r   c                    i | ]
\  }}d |f|S )r,  r1   )ra   rb   r   s      r3   r   z#test_accumulate.<locals>.<dictcomp>   s"    
9
9
9fq!FA;
9
9
9r6   r,  r  )r+   r.   r   rT   r$  r  r  )r  r   r,   r'   r      r      )r,   rF   r   r  rS        r+   rU   )r+   r.   r   )	enumeraterw   r   rP   
accumulater#   r;   rG   rz   r&   rx   )partsr-   r:   rZ  s       r3   test_accumulater    s   YYAQF+E
9
9	%(8(8
9
9
9C
sFCJJ''A	SA6Q\\#&&+++++6Q\\#r**/////99;;3333333<<R  ((**.J.J.JJJJJ<<  %%--//3L3L3LLLLL
222A<<$$&&)))333333r6   c            	     j   t          j        t          d          d          } |                     d dd          }t	          j        |j        |                                          }|D ]H}|D ]C} || ur=t          t          d|                    t          t          d|                     z  rJ DIt          j        t          d          d	          } |                     d
 d          }t          |j                  dk     sJ t	          j        |j        |                                          }|D ]H}|D ]C} || ur=t          t          d|                    t          t          d|                     z  rJ DIt          j        t          d          d          } |                     d dd          }t	          j        |j        |                                          }|D ]H}|D ]C} || ur=t          t          d|                    t          t          d|                     z  rJ DId S )N   rF   rU   c                    | dz  S r  r1   r2   s    r3   rO   z$test_groupby_tasks.<locals>.<lambda>  s
    a"f r6   tasks
max_branchshuffler   r~  rS   c                    | dz  S )N{   r1   r2   s    r3   rO   z$test_groupby_tasks.<locals>.<lambda>  s
    a#g r6   r  i'  iY  c                    | dz  S )Ni  r1   r2   s    r3   rO   z$test_groupby_tasks.<locals>.<lambda>"  s
    a$h r6      )
rw   rx   ry   r	   r~   r  r@   r   r   rP   )r:   r  
partitionsrY   s       r3   test_groupby_tasksr    s@   
s333A
))$$G)
D
DC#(C$5$5$7$788J ? ? 	? 	?AzzuQ{{++c%1++.>.>>>>>	? 	t#666A
))%%w)
7
7Csx==6!!!!#(C$5$5$7$788J ? ? 	? 	?AzzuQ{{++c%1++.>.>>>>>	? 	u3777A
))&&2w)
G
GC#(C$5$5$7$788J ? ? 	? 	?AzzuQ{{++c%1++.>.>>>>>	?? ?r6   c                 h   t          j        t          d          d          } d }d }t          |                     |dd          j                  t          |                     |dd          j                  k    sJ t          |                     |dd          j                  t          |                     |dd          j                  k    sJ t          |                     |dd          j                  t          |                     |dd          j                  k    sJ d S )	Nr  rF   rU   c                    | dz  S r  r1   r2   s    r3   rO   z*test_groupby_tasks_names.<locals>.<lambda>-  
    QV r6   c                    | dz  S )Nr   r1   r2   s    r3   rO   z*test_groupby_tasks_names.<locals>.<lambda>.  s
    a"f r6   r  r  r,   )rw   rx   ry   r   r	   r~   )r:   r   func2s      r3   test_groupby_tasks_namesr  +  sI   
s333ADEqyy!Wy==BCCs			$1g	66;H H     qyy!Wy==BCCs			$1g	66;H H     qyy!Wy==BCCs			%Aw	77<H H      r6   zsize,npartitions,groups))r~  r   rS   )i90     i  )rS   r+   r   c                    fd}t          j        | |                              |d          }|                    d          }t	          |          t          |t          |                     k    sJ d S )Nc                    | z  S r?   r1   )r)   groupss    r3   rO   z&test_groupby_tasks_2.<locals>.<lambda>>  s    QZ r6   rU   r  r  r   r   rw   ry   r	   rG   r	  )sizer<   r   r   r:   rI   s     `   r3   test_groupby_tasks_2r  :  sz      D
;///77g7NNAYYY((F<<74t55555555r6   c                     d } t          j        dd                              | dd          }|                    d	          }t	          |          t          | t          d                    k    sJ d S )
Nc                    | dz  S r  r1   r2   s    r3   rO   z&test_groupby_tasks_3.<locals>.<lambda>E  r  r6   r   r'   rU   r  r,   )r  r  r   r   r  )r   r:   rI   s      r3   test_groupby_tasks_3r  D  ss    D
###++D'a+PPAYYY((F<<74r33333333r6   c                    t                      5 } t          j        dd                              d                               t
                    }|                    t          j        	                    | d                     t          t          j        |                     dk    sJ 	 d d d            d S # 1 swxY w Y   d S )Nr'   rU   c                    | dk    S r!  r1   r2   s    r3   rO   z4test_to_textfiles_empty_partitions.<locals>.<lambda>N  s
    Q r6   z*.txt)r!   rw   ry   r   rz   r   rD  r  rE  r   rP   listdirr   s     r3   "test_to_textfiles_empty_partitionsr
  L  s    	 'QHQA&&&--.>.>??CCCHH	rw||Aw//0002:a==!!Q&&&&&' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' 's   BB::B>B>c                 $   t          j        t          d          d          } t          |                     d                                           d           t          |                     d                                           d           d S )NrT   rS   rU   c                    | dz  dk    S r0   r1   r2   s    r3   rO   z&test_reduction_empty.<locals>.<lambda>U      Q! r6   r   c                    | dz  dk    S r0   r1   r2   s    r3   rO   z&test_reduction_empty.<locals>.<lambda>V  r  r6   r   )rw   rx   ry   r   r   r   r&  rd   s    r3   test_reduction_emptyr  S  s{    
r444Aahh++,,0022A666ahh++,,0022A66666r6   )r+   r,   rF   c                @   t          j        g d|                               d           }t          |                    d          d           t          j        |                    d          |                    d          d          }|dk    sJ t          j        t                    5  t          j        g d	|           }|                    d                               d                              d           d d d            d S # 1 swxY w Y   d S )
Nr  rU   r,   r  r+   r   r   r  r  )
rw   rx   r   r   r&  rG   r   r   r   r   )r<   r:   valss      r3   test_reduction_empty_aggregater  Y  sY   
;???FFtLLAaeee""A&&&:aeee**AEEaE,@,@FSSSD6>>>>	z	"	" D D\\\{CCC	q))11F1CCCD D D D D D D D D D D D D D D D D Ds   0ADDDc                      e Zd Zd Zd ZdS )
StrictRealc                P    t          |t                    sJ | j        |j        k    S r?   r   r  realr  rL  s     r3   __eq__zStrictReal.__eq__e  '    %,,,,,yEJ&&r6   c                P    t          |t                    sJ | j        |j        k    S r?   r  r  s     r3   __ne__zStrictReal.__ne__i  r  r6   N)r  r  r  r  r  r1   r6   r3   r  r  d  s2        ' ' '' ' ' ' 'r6   r  c                     t          j        d t          d          D             d          } t          |                     t
          t
                    t          d                     d S )Nc                ,    g | ]}t          |          S r1   )r  r  s     r3   rc   z>test_reduction_with_non_comparable_objects.<locals>.<listcomp>o  s    ;;;A*Q--;;;r6   rT   r,   r  r   )rw   rx   ry   r   r   r   r  rd   s    r3   *test_reduction_with_non_comparable_objectsr  n  sV    
;;r;;;ANNNAaffS#
1.....r6   c                    t          j        d          t          j        fdt	          d          D             d          } fd}|                     ||                              d          j        d	k    sJ d S )
Nzscipy.sparsec                <    g | ]}                     d g          S r   )
csr_matrix)ra   r)   sps     r3   rc   z7test_reduction_with_sparse_matrices.<locals>.<listcomp>u  s'    ???"--,,???r6   rF   r,   r  c                2                         | |g          S r?   )vstack)rY   r:   r#  s     r3   	sp_reducez6test_reduction_with_sparse_matrices.<locals>.sp_reducew  s    yy!Q   r6   r   r   )rF   r+   )r   r:  rw   rx   ry   r   rG   shape)r:   r&  r#  s     @r3   #test_reduction_with_sparse_matricesr(  s  s    		^	,	,B
????eAhh???PQRRRA! ! ! ! ! 66)Y''//&/AAG6QQQQQQr6   c                 T    t          t          j        g                     g k    sJ d S r?   )r{   rw   rx   r1   r6   r3   
test_emptyr*  }  s,     $$%%++++++r6   c                    ddl m} m} t          j        t          d                    } | | |                    }|                                |                                k    sJ |                                } | | |                    }|                                |                                k    sJ d S )Nr   )dumpsloadsrS   )pickler,  r-  rw   rx   ry   rG   r}   )r,  r-  r:   r   ss2s         r3   test_bag_picklabler1    s    ########
s$$A	uuQxxB99;;"**,,&&&&	A	uuQxxB99;;"**,,&&&&&&r6   c                     t          j        ddig                              d          } |                     d          }t	          |          dddigik    sJ d S )NrY   r+   r   r   )rw   rx   r	   rG   r	  r  s     r3   test_msgpack_unicoder3    s`    
3($$,,S11AYYY((F<<Aaz?******r6   c                 X    d } t          j        | g          }t          || g           d S )Nc                     d S r?   r1   r1   r6   r3   rO   z/test_bag_with_single_callable.<locals>.<lambda>  s     r6   )rw   rx   r   )r   r:   s     r3   test_bag_with_single_callabler6    s3    A
!Aa!r6   c                 6   t          j        dd          } |                     t                    }|                    t                    }|                    |j        |                                          |j                                                                        z  rJ |                    |j        |                                |                                          t          fd|                                D                       sJ d S )NrT   r,   rU   )	fuse_keysc              3      K   | ]}|v V  	d S r?   r1   )ra   r-  r-   s     r3   r.  z*test_optimize_fuse_keys.<locals>.<genexpr>  s'      33AqCx333333r6   )	rw   ry   rz   r&   __dask_optimize__r~   r@   rB   r(  )r)   rp  r  r-   s      @r3   test_optimize_fuse_keysr;    s    
###A	c

A	c

A


afaoo&7&7
8
8Cv{{}}sxxzz))))


afaoo&7&71??CTCT

U
UC3333!2!23333333333r6   c                     d gfd} fd}t          d |             idd          }|                    |t                    }t          |t          t	          d                               d S )Nc               3  B   K   t          d          D ]} | d<   | V  d S )NrT   r   r  )rb   currents    r3   partz&test_reductions_are_lazy.<locals>.part  s:      r 	 	AGAJGGGG	 	r6   c                >    d         dk    sJ t          |           S r   )r}   )r?  r>  s    r3   r   z&test_reductions_are_lazy.<locals>.func  s     qzQ4yyr6   )r/  r   r/  r+   rT   )r   r0  r}   r   ry   )r?  r   r:   resr>  s       @r3   test_reductions_are_lazyrB    s    fG    
     	Z %++A
++dC
 
 Cc3uRyy>>"""""r6   c                     t          j        dd          } |                     d           }t          t          t          |                    t          t          t          |                    k    sJ d S )NrT   rF   rU   c                    | dz  S )Nr.   r1   r2   s    r3   rO   z'test_repeated_groupby.<locals>.<lambda>  s
    AE r6   )rw   ry   r	   r   rP   r	  ri   s     r3   test_repeated_groupbyrE    s`    
###A			//""A#tAww6#tAww#7#7777777r6   c           
        t          j        dd          }t          d          5 }t          j                            t          |           |          5  |                    d           }|                                 t          d t          j        t          |                     D                       sJ 	 d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )NrT   rF   rU   )temporary_directorypoolc                    | dz  S )Nr,   r1   r2   s    r3   rO   z*test_temporary_directory.<locals>.<lambda>  s
    QU r6   c              3  @   K   | ]}|                     d           V  dS )z.partdN)endswith)ra   r  s     r3   r.  z+test_temporary_directory.<locals>.<genexpr>  s.      OOr{{8,,OOOOOOr6   )rw   ry   r   r~   r  r   r   r	   rG   r'  r  r	  )r!   r:   rH  r   s       r3   test_temporary_directoryrL    st   
###A 
Q		 P4[__V4_HH 	P 	P??++BJJLLLOOrz#f++7N7NOOOOOOOOO	P 	P 	P 	P 	P 	P 	P 	P 	P 	P 	P 	P 	P 	P 	PP P P P P P P P P P P P P P P P P Ps6   /CA%C;CC	CC	CC#&C#c                    t          j        g           } t          |                     t                                                    d           t          |                     t                                                    d           t          |                     t                                                    d           t          |                     t                                                    d           d S )NTF)	rw   rx   r   rz   r&   r(  r'  r}   r   rd   s    r3   test_empty_bagrN    s    
AaeeCjjnn%%%aeeCjjnn&&&aeeCjjnn&&&aeeCjj  %(((((r6   c                 $   t          j        g dd          } |                     d          }|d                             d          sJ |d                             d          sJ t	          j        d           t	          j        d           d S )	Nr=  r,   rU   zfoo*r   foo0r+   foo1)rw   rx   rD  rK  r  r   )r:   pathss     r3   test_bag_pathsrS    s    
...A>>>ANN6""E8V$$$$$8V$$$$$IfIfr6   c                     d } t          j        g d          }t          |                    | d          g d           t          |                    | t	          j        d                    g d           d S )Nc                     fd| D             S )Nc                    g | ]}|z   S r1   r1   )ra   r)   r/  s     r3   rc   z?test_map_partitions_arg.<locals>.append_str.<locals>.<listcomp>  s    )))!A)))r6   r1   )r   r/  s    `r3   
append_strz+test_map_partitions_arg.<locals>.append_str  s    ))))y))))r6   rX   r/  )afoobfoocfoo)rw   rx   r   rd  r~   r   )rW  mybags     r3   test_map_partitions_argr\    s    * * * ___--Ee"":u557O7O7OPPPZe)<)<==?W?W?W    r6   c                    t          j        g d          } t          |                     t                                                              }dt          t          j        j        |          v sJ t          |                     t                                                              t          | 
                    t                                                              k    sJ d S )Nrv   r&   )rw   rx   r	  rz   r&   __dask_graph__r~   r3  	key_splitr   rd  r  s     r3   test_map_keynamesr`    s    
##AQUU3ZZ&&(())AC
,a000000quuSzz((**++s	,,..0 0      r6   c                     G d d          t          j                    fd} d }t          j        t	          d          d                              |                               |                              |                               |                                          }	 t          j                     |	                    d	           t          j
                     d S # t          j
                     w xY w)
Nc                      e Zd Zd ZdS )Ctest_map_releases_element_references_as_soon_as_possible.<locals>.Cc                    || _         d S r?   rb   )r  rb   s     r3   __init__zLtest_map_releases_element_references_as_soon_as_possible.<locals>.C.__init__  s    DFFFr6   N)r  r  r  rf  r1   r6   r3   Crc    s#        	 	 	 	 	r6   rg  c                r    t                    dk    sJ  |           }                    |           |S r   )rP   r#   )rb   rR  rg  	in_memorys     r3   f_createzJtest_map_releases_element_references_as_soon_as_possible.<locals>.f_create  s@    9~~"""" AaDDar6   c                    | j         dz   S )NrS   re  )rR  s    r3   f_dropzHtest_map_releases_element_references_as_soon_as_possible.<locals>.f_drop  s    sSyr6   r,   r+   rU   r   r   )weakrefWeakSetrw   rx   ry   rz   r}   gcdisablerG   enable)rj  rl  r:   rg  ri  s      @@r3   8test_map_releases_element_references_as_soon_as_possiblerr    s           
 !!I       
 	qq111	X	V	X	V	 

			F	###
		s   .)C, ,Dc                     t          j        d          } |                     dd                                          d         }t	          j        |          }|                                }t          |g d           d S )N
dask.arrayrT   r'   )chunksr   )      ?rv  rv  rv  rv  )r   r:  onesr*  rw   r   rG   r   )daobjr   r:   s       r3   test_bagged_array_delayedrz  &  sq    		\	*	*B
''"Q'


*
*
,
,Q
/C
/#

CAa***+++++r6   c                 @   t          j        ddgd          } |                                 | j        fk    sJ | j        j                                        | j        hk    sJ | j        j        | j        t                      ik    sJ | 	                                }|                                |j
        fk    sJ |j        j                                        | j        |j
        hk    sJ |j        j        | j        t                      |j
        | j        hik    sJ d S )Nr+   r,   rU   )rw   rx   r  r;   r~   layersrB   dependenciesr   r&  r   )rY   rb   s     r3   test_dask_layersr~  /  s	   
!QQ///A16)++++6=AF8++++616355/1111	A15(****6=AFAE?2222616355!%!&"BBBBBBBr6   r   FTc                   t          j        d          }t          j                            |                    d                                          d                   }|j        d         }|j        dd          dk    sJ |j        j	        
                                d|z   hk    sJ |j        j        d|z   t                      ik    sJ |                                d|z   fk    sJ |                    d          dz   }|                    |           d         }t          j                            |          }|j        |j        k    sJ |j        |j        u sJ |                                |                                k    sJ |                    |           }|                                |                                k    sJ | sb|j        |j        u sJ t          j        t          d          5  t          j        |j        |j                   d d d            n# 1 swxY w Y   t          j        t          d          5  t          j        |j        |j        fd	
           d d d            d S # 1 swxY w Y   d S )Nrt  r+   r   r   zdelayed-r  znot in)rm  r/  rz  )r   r:  rw   r  r   rw  r*  r   r~   r|  rB   r}  r   r  r   r   r;   )r   rx  rb   r;   arrr   backs          r3   test_dask_layers_to_delayedr  :  s    
	\	*	*B
RWWQZZ2244Q788A58D596=J$5#666666:#4cee"<<<<<:#4"66666
''!**q.CnnHn55a8G
W%%A5GK6W\!!!!'"9"9";";;;;;<<x<00D!!Q%6%6%8%88888 )yCH$$$$]:X666 	) 	)GDItx(((	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 
z	2	2	2 4 4
38+U33334 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4s$   : H&&H*-H*#I<<J J c                    t          j        d           ddlm}  ddlm} t          j        ddddd	dgd	
          }t          j        d          5  |	                    d           }|	                    d           }|	                    d           }d d d            n# 1 swxY w Y   t          |j                  |j        dz  k    sJ |                                }t          |j                  t          |j                  |j        t           |                      z  z   k     sJ |                    d          }t          |j                                        |j                                        z
            |j        dt           |                      z   z  k    sJ t          |j        d          j        ddik    sJ  | ||           d S )Nr  r   r   )pyarrow_strings_enabledtest1r+   )r;   v1test2r,   rU   T)r/  c                4    t          di | d| d         dz   iS )Nv2r  r+   r1   r	  r]   s    r3   rO   z2test_to_dataframe_optimize_graph.<locals>.<lambda>e  '    D5515541555 r6   c                4    t          di | d| d         dz   iS )Nv3r  r+   r1   r  r]   s    r3   rO   z2test_to_dataframe_optimize_graph.<locals>.<lambda>f  r  r6   c                4    t          di | d| d         dz   iS )Nv4r  r+   r1   r  r]   s    r3   rO   z2test_to_dataframe_optimize_graph.<locals>.<lambda>g  r  r6   rF   Fr  r/  )r   r:  dask.dataframe.utilsr   r  rw   rx   r~   annotaterz   rP   r<   r  r%  rB   r%   r   )assert_eq_dfr  r)   rp  r   r  s         r3    test_to_dataframe_optimize_graphr  Z  s6   
()))>>>>>><<<<<<
	#	#gQ%?%?@a	 	 	A
 
4	 	 	  7 7EE5566EE5566EE55667 7 7 7 7 7 7 7 7 7 7 7 7 7 7 qv;;!-!+++++ 	
A qv;;QVq}s;R;R;T;T7U7U'UUUUUU 
u	-	-B rw||~~-..!-	C''))***3    
 !!,,8UDMIIIILBs   AB##B'*B'nworkers)rS      i  r~  c                    d}d}|| k     rkt          t          j        t          |                    j                  }|t          j        t          d| dz                      z  }|dk     sJ || k     id S d S )Nr   r+   rT   i N  )rP   rw   rx   ry   r~   r@  floorr   )r  ntasksnitemss      r3   +test_default_partitioning_worker_saturationr    s    
 FF
8

R%eFmm449::$*SHrM22333 8





r6   c                    t          | d| z  t          dt          j        | dz                                D ]=}t	          t          j        t          |          |           j                  | k    sJ >d S )NrT   r+   rU   )ry   r   r@  r  rP   rw   rx   r~   )r  r  s     r3   test_npartitions_saturationr    sz     "x-Q
8b=8Q8Q1R1RSS 
 
 vHEEEJKKxWWWWW
 
r6   c                 n   t          j        t          dd          d          } t          |                     t
                                                              }|                     d           }t          |                    t
                                                              }||k    sJ dS )z)https://github.com/dask/dask/issues/10338r+   rS   r.   rU   c                    | S r?   r1   r2   s    r3   rO   z*test_map_total_mem_usage.<locals>.<lambda>  s     r6   N)rw   rx   ry   r}   rd  r   rG   rz   )r:   total_mem_brZ   total_mem_cs       r3   test_map_total_mem_usager    s    
q#A666Aa&&77??AABBK	kkAa&&77??AABBK+%%%%%%r6   )r~  )
__future__r   ro  r@  r  rj  r  rm  r;  r   collections.abcr   concurrent.futuresr   gzipr   	itertoolsr   r  r   tlzr	   r
   r   r   r   r   r   r~   dask.bagr   rw   dask.bag.corer   r   r   r   r   r   r   r   r   r   r   dask.bag.utilsr   dask.blockwiser   r  r   dask.typingr   
dask.utilsr    r!   r"   dask.utils_testr#   r$   r%   r&   ry   r-   __annotations__r{   r   r:   r4   r8   r=   rC   rJ   rQ   r   r   r   r   r   r   markparametrizer   reprr   r   r   r   r   r   r   r   r  r  r  r  r  r"  r)  r1  r4  r7  r=  rC  rF  r   rM  rV  ra  re  rv  r|  r  r  r  r  r  r  r   r  r  r  r  r  r  r  r  slownetworkskipr  r  r  r  r  xfailr  r  r  r  r  r  r	  r  r  r  r  r  r   r$  r:  r  r]  rM  rW  rZ  r^  rc  rg  ry  r{  r}  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r
  r  r  r%  r  r  r(  r*  r1  r3  r6  r;  rB  rE  rL  rN  rS  r\  r`  rr  rz  r~  r  r  r  r  r  r1   r6   r3   <module>r     s   " " " " " " " 				  				          $ $ $ $ $ $ 2 2 2 2 2 2               E E E E E E E E E E E E E E E E E E                                 % $ $ $ $ $ $ $ $ $ $ $                   1 1 1 1 1 1 1 1 1 1 F F F F F F F F F F F F
uajUAJOO O O O ODqNNQCS!      
3 3 3B B B- - -14 14 14hU U U(N N N"
+ 
+ 
+& & &E E E
 #t--? ? .-?8 8 84 4 4     O O O<& & &7 7 7< < <  $V V V
, , , A//: : 0/:D D D
< < <& & &D D D     * 			22! ! 32! B00% % 10%, , ,
! ! !
 (DL*V*VW H H HQ Q Q. . ."U U U8 8 8>( ( (: : :    B B B  	 	 	 K  5 5 5(( ( (5 5 5! ! !   = = =3 3 3 !!# # "!  #K K K
" 
" 
"" " "5 5 5  DEE    FE2% % %  . . .' ' 'K K K5 5 5Q Q Q# # #8 8 8, , ,0 0 0( ( (* * *: : :z 8ug.T
; x00  10  @. . .0  $  "L L LC C C&P P P
  	- 	- 	-, , ,	  	  	  	  	  	  	  	 = = =: : :     
	 	 	$ $ $8< < <   + + + 77F F F 87F  1 1 122!2!2!233  43 32 
 
 
   
 
 
	 	 	
 
 
> > >4 4 4? ? ?<   RRR 6 6 64 4 4' ' '7 7 7 			22D D 32D' ' ' ' ' ' ' '/ / /
R R R, , ,	' 	' 	'+ + +  	4 	4 	4# # #&8 8 8	P 	P 	P) ) )  	 	 	  1 1 1h, , ,C C C eT]334 4 434>$ $ $N %:%:%:;;	 	 <;	 %:%:%:;;
 
 <;
& & & & &r6   