
    >ieg                      d dl mZ d dlZd dlZd dlZd dlZd dlm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZ d dlmZmZmZmZmZmZ d dlmZ d d	lmZ d d
lmZm Z m!Z!m"Z"m#Z# d dl$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+ d dl,m-Z-m.Z. 	 d dl/Z0n# e1$ r dZ0Y nw xY wd Z2d Z3d Z4d Z5d Z6ej7        8                    dg d          d             Z9d Z:d Z;d Z<d Z=d Z>ej7        8                    dddg          ej7        8                    dg d          d                         Z?ej7        8                    dddg          ej7        8                    dg d          ej7        8                    d ddg          d!                                     Z@d" ZAd# ZBd$ ZCd% ZDd& ZEd' ZFej7        8                    d(d)d*g          d+             ZGd, ZHej7        8                    d-d.d/g          d0             ZIej7        8                    d-d.d/g          d1             ZJej7        8                    d-d.d/g          d2             ZKd3 ZLd4 ZMej7        N                    ed5d6          ej7        8                    d7d8d9d: ejO        d;          fd<d= ejO        d;          fd> ejO        d;          fg          d?                         ZPej7        8                    dg d@          ej7        8                    dAddg          dB                         ZQej7        8                    dg d@          dC             ZRej7        8                    ddDd/g          dE             ZSdF ZTej7        8                    dGg dH          ej7        8                    ddIdJg          ej7        8                    dK ejU        dLej7        N                    dMN          O           ejU        dPej7        jV        O          g          dQ                                     ZWdR ZXej7        jY        ej7        8                    dg d@          dS                         ZZej7        [                    edTN          ej7        jY        ej7        8                    dg d@          dU                                     Z\dV Z]ej7        jY        ej7        8                    dg d@          dW                         Z^ej7        8                    dXi  e_dYZ           e_dY[           e_d\dYgd]^           e_d_d\dYgd`a           e_d\d_g[           e_d_dbg[          g          dc             Z`dd Zaej7        8                    deddg          df             Zbdg Zcej7        8                    ddhdDg          ej7        8                    deddg          di                         Zddj Zedk Zfdl Zgdm Zhdn Zido Zjdp Zkdq Zldr Zmds Zndt Zodu Zpej7        8                    dvdw ejU        dddej7        N                    epe e	dx          k    dyeqdz          O          d{d| ejU        dddej7        N                    epe e	dx          k    dyeqdz          O          d}g          d~             Zrd Zsd Ztd Zuej7        [                    edN          d             Zvej7        [                    edN          d             Zwej7        [                    edN          d             Zxej7        [                    edN          d             Zyd Zzd Z{d Z|d Z}d Z~ej7        8                    dKdL ejU        dPej7        jV        O          g          d             Zej7        8                    dddg          d             Zej7        8                    ddd ejU        dej7        [                    e0du pe dN          O           ejU        dej7        [                    e0du pe dN          O          g          d             Zd Zd Zd Zd Zd Zej7        8                    dg d          ej7        8                    dddg          ej7        8                    dddg          d                                     Zej7        8                    dddg          d             Zej7        8                    ddDd.g          d             Zej7        8                    ddhd/g          ej7        8                    dg d          ej7        8                    dg d          d                                     ZdS )    )annotationsN)Version)is_object_dtype)
PY_VERSION)compute_as_if_collection)PANDAS_GE_140PANDAS_GE_150PANDAS_GE_200PANDAS_GE_210PANDAS_GE_220tm)_Frame)concat)_maybe_align_partitionsalign_partitionsconcat_indexed_dataframes	hash_joinmerge_indexed_dataframes)assert_divisions	assert_eq
check_metaclear_known_categoriesget_string_dtypehas_known_categories	make_meta)	hlg_layerhlg_layer_topologicalc            	     6   t          j        g dt          d          dg d          } t          j        | g d          }t          j        g dt          d          dg d	          }t          j        |g d
          }t          j                            ddidd          }t          ||          \  \  }}fd} |||||           |j        df|j        dfg|j        df|j        dfg|j        df|j        dfg|j        df|j        dfg|j        df|j        dfggk    sJ t          |||          \  \  }}} |||||           |j        dfd |j        dfg|j        dfd |j        dfg|j        dfd |j        dfg|j        dfd |j        dfg|j        dfd |j        dfggk    sJ t          |d           t          j        g dg dd          }	t          j        g dg dd          }
t          j
        |	d          t          j
        |
d          ft          j
        |	d          t          j
        |
d          ft          j
        |	d          t          j
        |
d          ft          j
        |	d          t          j
        |
d          ffD ]<\  }}t          ||          \  \  }}}}t          ||	           t          ||
           =t          j        g dg ddt          d                    }	t          j        g dg ddt          d                    }
t          j
        |	d          t          j
        |
d          ft          j
        |	d          t          j
        |
d          ft          j
        |	d          t          j
        |
d          ft          j
        |	d          t          j
        |
d          ffD ]<\  }}t          ||          \  \  }}}}t          ||	           t          ||
           =d S )N                  abdabdxy
         (   2   <   index)r+   r.   r0   r    r!   r"   r#   abda)r-   F   P   d   )r-   r6   r7   )sr   r+   r8   i8c                   t          | t          j                  sJ t          |t          j                  sJ t          |t          j                  sJ t          |t          j                  sJ t          | |           t          ||           dk    sJ t          t                    sJ t                    dt                    z   k    sJ d S )N)r+   r-   r.   r0   r6   r7   r    )
isinstancedd	DataFramer   listlen)abaabbL	divisionss       ?lib/python3.11/site-packages/dask/dataframe/tests/test_multi.py_checkz%test_align_partitions.<locals>._check=   s    !R\*****!R\*****"bl+++++"bl+++++!R!R55555!T"""""9~~SVV++++++    r   r    r!   r"   r#   r    r!   r"   r#   r$   r%      rJ   r%   r$   r#   r"   r!   r    r@   rA   cdabcdefgfghijkl)pdr=   r>   r<   repartitioncoreScalarr   _namer   from_pandas)Ar@   BrA   r8   rB   rC   rG   ssldfrdflhsrhslresultrresultdivpartsrD   rE   s                    @@rF   test_align_partitionsrc   0   s   
   tH~~66>V>V>V	 	 	A 	q,,,''A
<<<d6ll;;CTCTCTUUUA
q---((A
"~sD11A-a33HRi	, 	, 	, 	, 	, 	, F1aR
(A1&
(A1&
(A1&
(A1&
(A1&     "2!Q!:!:LRR)Q
F1aR
(Arxm,
(Arxm,
(Arxm,
(Arxm,
(Arxm,     b"
,2229N9N9NOO
P
PC
,2229N9N9NOO
P
PC 
Q		Q!7!78	Q		Q!7!78	Q		Q!7!78	Q		Q!7!78	    S *:#s)C)C&'C'3'3 ,###*?*?*?@@Y  C ,###*?*?*?@@Y  C
 
Q		Q!7!78	Q		Q!7!78	Q		Q!7!78	Q		Q!7!78	    S *:#s)C)C&'C'3'3   rH   c                 4   t          j        g dg dd          } t          j        | d          }t          j        | dd          }|j        rJ t          j        t                    5  t          ||           d d d            n# 1 swxY w Y   t          j        | dz   dd          }t          j        | dd          }|j        rJ |j        rJ t          j        t                    5  t          ||           d d d            d S # 1 swxY w Y   d S )	NrI   rK   rL   r!   npartitionsFrf   sortr    )	rR   r=   r<   rW   known_divisionspytestraises
ValueErrorr   dfddfddf2s      rF   'test_align_partitions_unknown_divisionsrq   |   s   	1118M8M8MNN	O	OB
.
+
+
+C>"!%888D####	z	"	" $ $d###$ $ $ $ $ $ $ $ $ $ $ $ $ $ $ .aQU
;
;
;C>"!%888D""""####	z	"	" $ $d###$ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $s$   +BBB/DDDc                 F   t          j        g dg dd          } t          j        | dz   d          }t          j        | d          }t	          ||g          \  }}||u sJ ||u sJ t          j        | dz   dd          }t          j        | dd          }|j        rJ |j        rJ t	          ||g          \  }}||u sJ ||u sJ t          j        | dz   d          }t          j        | d	          }t	          ||g          \  }}|j        |j        k    sJ t          j        | dz   dd          }t          j        | d	d          }|j        rJ |j        rJ t          j        t                    5  t	          ||g           d d d            n# 1 swxY w Y   t          j        | d          }t          j        | dd          }|j        rJ t          j        t                    5  t	          ||g           d d d            d S # 1 swxY w Y   d S )
NrI   rK   rL   r    r!   re   Frg   r"   )
rR   r=   r<   rW   r   ri   rE   rj   rk   rl   )rn   ro   rp   r@   rA   s        rF   test__maybe_align_partitionsrs      s   	1118M8M8MNN	O	OB
.aQ
/
/
/C>"!,,,D"C;//DAq88889999 .aQU
;
;
;C>"!%888D""""####"C;//DAq88889999 .aQ
/
/
/C>"!,,,D"C;//DAq;!+%%%% .aQU
;
;
;C>"!%888D""""####	z	"	" - -d,,,- - - - - - - - - - - - - - - .
+
+
+C>"!%888D####	z	"	" - -d,,,- - - - - - - - - - - - - - - - - -s$   ?FF!$F!7HHHc                    t          j        dg dig d          } t          j        | g d          }t          j        dt	          d          ig d          }t          j        |g d	          }t          ||d
          }|j        d         |j        d         k    sJ |j        d         t          |j        |j        z             k    sJ t          || 	                    |                     t          ||d          }|j        d         |j        d         k    sJ |j        d         |j        d         k    sJ t          || 	                    |d                     t          ||d          }|j        d         dk    sJ |j        d         t          |j        |j        z             k    sJ t          |
                                | 	                    |d                     t          ||d          }|j        d         dk    sJ |j        d         dk    sJ t          |
                                | 	                    |d                     t          t          ||d          j                  t          t          ||d          j                  k    sJ t          t          ||d          j                  t          t          ||d          j                  k    sJ d S )Nr(   r   r    r!   r"   r#   r%   rJ   r1   r    r#   rJ   r)   abcdefr    r!   r#   r$   r%      r    r!   r$   ry   lefthowr   rightinnerr    outerry   )rR   r=   r<   rS   r>   r   rE   maxr   joincomputesorteddask)rX   r@   rY   rA   rN   s        rF   1test_merge_indexed_dataframe_to_indexed_dataframer      s   
c---.6H6H6HIIIA
q)))$$A
c4>>*2D2D2DEEEA
q,,,''A A6222A;q>Q[^++++;r?c!+";<<<<<<a A7333A;q>Q[^++++;r?ak"o----aw''((( A7333A;q>Q;r?c!+";<<<<<<aiikk166!611222 A7333A;q>Q;r?aaiikk166!611222*1aW===BCCv A73338H H     *1aW===BCCv A73338H H      rH   c                   t          | t          j                  r|                     d          }n| }t          |t          j                  r|                    d          }n|}t	          j        |j        |j                   t          |t          j                  rY|                    t          |j                            j
        }|                    t          |j                            j
        }n2|                                j
        }|                                j
        }t          j                            ||           d S )Nsync	scheduler)r;   r<   r=   r   r   assert_index_equalcolumnsrR   sort_valuesr>   values_compatassert_numpy_array_equal)rB   rC   r@   rA   avbvs         rF   list_eqr      s   "bl## JJJ(("bl## JJJ((!)QY///!R\"" $]]4	??++2]]4	??++2]]__#]]__#J''B/////rH   r}   )r   r{   r   r   c           	        t          j        g dg dd          }t          j        |g d          }t          j        g dg dd          }t          j        |g d          }t	          |d	|d	|           }t          |j        d
                                          rJ |                                }t          j	        ||| d	          }t          ||           t	          |d|ddd|          }t          |j        d
                                          rJ |j        dk    sJ |                    d          }t          j	        ||dd dd          }t          ||           t	          |d	|d	d|          j        t	          |d	|d	d|          j        k    sJ t	          |d	|d	d|          j        t	          |d	|d	d|          j        k    sJ d S )Nr   r    r    r!   r!   r"   r#   r'   r   r#   r$   r    r"   r#   r#   r$   r%   r%   r$   r#   r"   r!   r    r)   zr   r!   r$   r)   r~   r(   r   r   r"   )rf   shufflezsingle-threadedr   r   )r   )rR   r=   r<   rS   r   r   r   is_materializedr   merger   rf   rV   )	r}   shuffle_methodrX   r@   rY   rA   rN   resultexpecteds	            rF   test_hash_joinr      s   
---4F4F4FGGHHA
q)))$$A
---4F4F4FGGHHA
q)))$$A!S!S#&&A$QVR00@@BBBBBYY[[Fx1c3''HFH 	!S!S'q.QQQA$QVR00@@BBBBB=AYY!2Y33Fx1gtS#66HFH 	!S!S'>BBBHQQWnEEEK	L 	L 	L 	L 	!S!S'>BBBHQQWnEEEK	L 	L 	L 	L 	L 	LrH   c                    t          j        t          t          d                    g dd          } t          j        t          t          d                    g dd          }t          j        t          t          dd                    g d	d
          }|                     |ddd          }|                    |ddd          }t          j        | d          }t          j        |d          }t          j        |d          }|                    |ddd          }|                    |ddd          }	t          ||	           d S )Nr%   )A0A1A2A3A4A5)keyrX   r#   B0B1B2B3)r   rY   r    r$   C0C1C2C3)r   Cr   _l_rr}   lsuffixrsuffixr"   re   r!   )rR   r=   r>   ranger   r<   rW   r   )
df1df2df3join_pdmulti_join_pdddf1rp   ddf3join_ddmulti_join_dds
             rF   test_sequential_joinsr     sC   
,U1XX%I%I%IJJ C ,tE!HH~~4L4L4LMM
N
NC
,tE!QKK007O7O7OPP
Q
QChhsthDDGLL'4LNNM >#1---D>#1---D>#1---Dii'4iFFGLL7D$LOOMm]+++++rH   c                 z   t          j        dt          d          ig d          } t          j        | d          }t          j        dt          d          ig d	          }t          j        |d
          }t          j        | |dd          }t          j        ||dd          }t          ||           d S )Nleft_valabcdabcdabcd)r    r"   rJ   	   r+            r,            r1   r#   re   	right_valxyzxyzxyzxyz)r    r!   r"   r%   rJ   r+      r               r"   T
left_indexright_index)rR   r=   r>   r<   rW   
merge_asofr   rX   r@   rY   rA   r   rN   s         rF   test_merge_asof_indexedr   -  s    
	T*%%&:::	 	 	A 	qa(((A
	d9oo&999	 	 	A 	qa(((A
at>>>A
at>>>AaOOOOOrH   c                 J   t          j        g dg dd          } t          j        | d          }t          j        g dg dd          }t          j        |d          }t          j        | |d	          }t          j        ||d	          }t          ||d
           d S )Nr    r$   r+   r@   rA   rN   r@   r   r!   re   r    r!   r"   r%   rJ   r@   r   r@   onFcheck_indexrR   r=   r<   rW   r   r   r   s         rF   test_merge_asof_on_basicr   ?  s    
:::???CCDDA
qa(((A
???IIJJA
qa(((A
as###A
as###Aa&&&&&&rH   c                    t          j        g dg dd          } t          j        | d          }t          j        g dg dd          }t          j        |d          }t	          j        t          d	          5  t          j        ||d
d           d d d            n# 1 swxY w Y   t	          j        t          d	          5  t          j        ||d
d           d d d            n# 1 swxY w Y   t	          j        t          d	          5  t          j        ||d
dd           d d d            d S # 1 swxY w Y   d S )Nr   r   r   r!   re   r   r   combination of bothmatchr@   r   )r   left_onr   )r   right_on)r   r   r   rR   r=   r<   rW   rj   rk   rl   r   rX   r@   rY   rA   s       rF   'test_merge_asof_on_lefton_righton_errorr   K  s   
:::???CCDDA
qa(((A
???IIJJA
qa(((A	z)>	?	?	? 8 8
asJ77778 8 8 8 8 8 8 8 8 8 8 8 8 8 8	z)>	?	?	? : :
as[9999: : : : : : : : : : : : : : :	z)>	?	?	? N N
asJMMMMN N N N N N N N N N N N N N N N N Ns6   >B##B'*B'	C..C25C2D;;D?D?c            	        t          j        g dg dg dd          } t          j        | d          }t          j        g dg dg dd	          }t          j        |d          }t	          j        t          d
          5  t          j        ||ddd           d d d            n# 1 swxY w Y   t	          j        t          d
          5  t          j        ||ddd           d d d            n# 1 swxY w Y   t	          j        t          d
          5  t          j        ||dddd           d d d            d S # 1 swxY w Y   d S )Nr   )r"   r%   r   r   )r@   rA   r   r!   re   r   )r"   r%   r   r      )r@   rA   r   r   r   r@   rA   r   )r   byleft_byr   )r   r   right_by)r   r   r   r   r   r   s       rF   'test_merge_asof_by_leftby_rightby_errorr   Y  sW   
:::III???SSTTA
qa(((A
oo$5$5$5OOOTT	 	A 	qa(((A	z)>	?	?	? @ @
assJ????@ @ @ @ @ @ @ @ @ @ @ @ @ @ @	z)>	?	?	? B B
ass[AAAAB B B B B B B B B B B B B B B	z)>	?	?	? V V
assJUUUUV V V V V V V V V V V V V V V V V Vs6   B**B.1B.C66C:=C:EEEallow_exact_matchesTF	direction)backwardforwardnearestc                R   t          j        g dg dd          }t          j        |d          }t          j        g dg dd          }t          j        |d          }t          j        ||d| |	          }t          j        ||d| |	          }t          ||d
           d S )Nr   r   r   r!   re   r   r   r@   )r   r   r   Fr   r   )r   r   rX   r@   rY   rA   r   rN   s           rF   test_merge_asof_onr   i  s     	:::???CCDDA
qa(((A
???IIJJA
qa(((A
	1*=	 	 	A 		1*=	 	 	A a&&&&&&rH   unknown_divisionsc           
     z   t          j        g dg ddg d          }t          j        |d          }t          j        dg d	ig d	          }t          j        |d          }|rd
t	          |j                  z  |_        t          j        ||dd| |          }t          j        ||dd| |          }t          ||           dD ]}	g dg dfg dg dfg dg dfg dg dffD ]\  }
}t          j        d|
id |
D                       }t          j        ||	          }t          j        d|i|          }t          j        ||	          }|rd
t	          |j                  z  |_        t          j        ||dd| |          }t          j        ||dd| |          }t          ||           Όd S )Nr   r   r   r+   r,   r-   r1   r!   re   r   )r!   r"   r%   rJ   Nr@   T)r   r   r   r   r    r!   r"   )r$   r+   r   r,   r3   )r$   r$   r+   r+   r   r   )	r#   r$   r%   r   r+      r   r   r   )r$   r+   r   )r#   r#   r$   r$   r%   r%   r   r   r+   r+   r  r  c                    g | ]}|d z  S )r+    ).0r(   s     rF   
<listcomp>z7test_merge_asof_left_on_right_index.<locals>.<listcomp>  s    .B.B.B!q2v.B.B.BrH   rA   )rR   r=   r<   rW   r?   rE   r   r   )r   r   r   rX   r@   rY   rA   r   rN   npartsa1idx2s               rF   #test_merge_asof_left_on_right_indexr  {  sY    	:::???CC<<<XXXA
qa(((A
k<<<0EEEA
qa(((A 1AK 0 00
		/	 	 	A 			/	 	 	A aOOO  __lll+\\???+###%E%E%EF[[BBBC	
 	 	HB c2Y.B.Br.B.B.BCCCAqf555Ac4[555Aqf555A  9%AK(8(88 $7#  A  $7#  A aOOOO=	 rH   c                 N   t          j        dg dig d          } t          j        | d          }t          j        dg dig d          }t          j        |d          }t          j        | |d	d	
          }t          j        ||d	d	
          }t          ||           d S )Nr   r   r   r1   r!   re   r   r   Tr   r   r   s         rF   &test_merge_asof_indexed_two_partitionsr    s    
j///2***EEEA
qa(((A
k???3???KKKA
qa(((A
at>>>A
at>>>AaOOOOOrH   c                    d dD             } g d}g d}g d}d dD             }g d}g d	}g d
}t          j        | |||dg d          }t          j        |d          }	t          j        ||||dg d          }
t          j        d |
j        dd         |
j        dd         gg d          }t          j        |
|dd          }t          j        ||	dd          }t          ||d           d S )Nc                6    g | ]}t          j        |          S r  rR   to_datetimer  rO   s     rF   r	  z)test_merge_asof_on_by.<locals>.<listcomp>  2        	q  rH   2016-05-25 13:30:00.023r  z2016-05-25 13:30:00.030z2016-05-25 13:30:00.0412016-05-25 13:30:00.048z2016-05-25 13:30:00.049z2016-05-25 13:30:00.072z2016-05-25 13:30:00.075GOOGMSFTr  r  r  AAPLr  r       @皙I@g\(I@gQI@r  g(\X@r  gzGJ@=
ףp@g{GI@g=
ףpI@g      J@r!  gq=
ףX@gףp=
@gp=
J@c                6    g | ]}t          j        |          S r  r  r  s     rF   r	  z)test_merge_asof_on_by.<locals>.<listcomp>  2     	 	 	 	q	 	 	rH   r  z2016-05-25 13:30:00.038r  r  r  r  r  r  r  r  r  r  g\(@g(\@g     X@K      r7   r7   r7   timetickerbidaskr   r#   re   r+  r,  pricequantityc                    | S r  r  r(   s    rF   <lambda>z'test_merge_asof_on_by.<locals>.<lambda>      ! rH   r   r!   r$   r   r!   r#   rE   r+  r,  )r   r   Fr   )rR   r=   r<   rW   from_mapilocr   r   times_A	tickers_Abids_Aasks_Atimes_B	tickers_Bprices_Bquantities_BrX   r@   rY   rA   r   rN   s                 rF   test_merge_asof_on_byrD    s    	
  G QPPIHHHFHHHF	 	
	 	 	G 988I444H+++L
IfVLL000	 	 	A 	qa(((A
$		
 	
 877	 	 	A 		
!afQqSk"))	 	 	A 	av(333A
av(333Aa&&&&&&rH   c            	     B   d dD             } g d}g d}g d}d dD             }g d}g d	}g d
}t          j        | |||dg d          }t          j        |d          }	t          j        ||||dg d          }
t          j        d |
j        dd         |
j        dd         gg d          }t          j        |
|ddt          j        d                    }t          j        ||	ddt          j        d                    }t          ||d           d S )Nc                6    g | ]}t          j        |          S r  r  r  s     rF   r	  z3test_merge_asof_on_by_tolerance.<locals>.<listcomp>  r  rH   r  r  r  r   c                6    g | ]}t          j        |          S r  r  r  s     rF   r	  z3test_merge_asof_on_by_tolerance.<locals>.<listcomp>  r#  rH   r$  r%  r&  r'  r*  r/  r#   re   r0  c                    | S r  r  r4  s    rF   r5  z1test_merge_asof_on_by_tolerance.<locals>.<lambda>2  r6  rH   r   r!   r$   r7  r8  r+  r,  2ms)r   r   	toleranceFr   	rR   r=   r<   rW   r9  r:  r   	Timedeltar   r;  s                 rF   test_merge_asof_on_by_tolerancerM    s    	
  G QPPIHHHFHHHF	 	
	 	 	G 988I444H+++L
IfVLL000	 	 	A 	qa(((A
$		
 	
 877	 	 	A 		
!afQqSk"))	 	 	A 	av(bl5>Q>QRRRA
av(bl5>Q>QRRRAa&&&&&&rH   c            	     F   d dD             } g d}g d}g d}d dD             }g d}g d	}g d
}t          j        | |||dg d          }t          j        |d          }	t          j        ||||dg d          }
t          j        d |
j        dd         |
j        dd         gg d          }t          j        |
|ddt          j        d          d          }t          j        ||	ddt          j        d          d          }t          ||d           d S )Nc                6    g | ]}t          j        |          S r  r  r  s     rF   r	  zDtest_merge_asof_on_by_tolerance_no_exact_matches.<locals>.<listcomp>=  r  rH   r  r  r  r   c                6    g | ]}t          j        |          S r  r  r  s     rF   r	  zDtest_merge_asof_on_by_tolerance_no_exact_matches.<locals>.<listcomp>M  r#  rH   r$  r%  r&  r'  r*  r/  r#   re   r0  c                    | S r  r  r4  s    rF   r5  zBtest_merge_asof_on_by_tolerance_no_exact_matches.<locals>.<lambda>l  r6  rH   r   r!   r$   r7  r8  r+  r,  10msF)r   r   rJ  r   r   rK  r;  s                 rF   0test_merge_asof_on_by_tolerance_no_exact_matchesrS  <  s    	
  G QPPIHHHFHHHF	 	
	 	 	G 988I444H+++L
IfVLL000	 	 	A 	qa(((A
$		
 	
 877	 	 	A 		
!afQqSk"))	 	 	A 			,v&&!	 	 	A 			,v&&!	 	 	A a&&&&&&rH   c                    t          j        g dg dd          } t          j        | d          }t          j        g dg dd          }t          j        |d          }t          j        ||d	
          }t          j        t          d          5  |                                 d d d            d S # 1 swxY w Y   d S )Nr   r   r   r!   re   )r!   r    r"   r%   rJ   r   r   r@   r   z
right keysr   )	rR   r=   r<   rW   r   rj   rk   rl   r   )rX   r@   rY   rA   r   s        rF   test_merge_asof_unsorted_raisesrU    s   
:::???CCDDA
qa(((A
???IIJJA
qa(((A]1aC(((F	z	6	6	6                   s   B77B;>B;c                    t          j        t          t          d                    t          t          d                    d          } t	          j        | d          }| | j        dk                                                                  ddi          }t	          j        |d          }t	          j	        |
                    d	          |
                    d	          d
d
          }t          j	        | 
                    d	          |
                    d	          d
d
          }t          ||d           t	          j	        |
                    d	          |
                    d	          d
d
          }t          j	        |
                    d	          | 
                    d	          d
d
          }t          ||d           t	          j	        |
                    d	          |
                    d	          d
d
          }t          j	        |
                    d	          |
                    d	          d
d
          }t          ||d           d S )Nr+   )	index_colgood_valr!   re   r   rX  	empty_valr/  rW  Tr   Fr   )rR   r=   r>   r   r<   rW   rW  copyrenamer   	set_indexr   )good_dfgood_ddempty_dfempty_dd	result_dd	result_dfs         rF   test_merge_asof_with_emptyrc    sZ   leBiid5QS99ooVVWWGnW!444G!A%&++--44j+=V4WW  ~hA666H +&&;''	  I +&&;''	  I i6666;''+&&	  I ;''+&&	  I i6666;'';''	  I ;'';''	  I i666666rH   zleft_col, right_col)	endofweek	timestamp)rd  rd  c                J   t          j        | g ddg di          }t          j        |g ddg di          }t          j        ||| |          }t          j        t          j        |d	          t          j        |d	          | |          }t          ||d
           d S )Nr   GroupCol)  !  rh  ri  rh  ri  )r   r   r    r"   GroupVal)rh  rh  ri  rh  r   r   r!   re   Fr   )rR   r=   r   r<   rW   r   )left_col	right_colr   r   rb  ra  s         rF   test_merge_asof_on_left_rightrn    s     ,(((<<<	
 C ,	<<<=U=U=UV
W
WC c39MMMI 
s***
s***	  I i666666rH   c                 J   t          j        t          t          j        d          gdz  g d                    } t          j        | | d          }t          dd          D ]?}t          j        | |	          }t          j        ||d          }t          ||           @d S )
Nz1-1-2020r"   r  )tsfoorp  )r{   r   r   r    r$   re   )	rR   r=   dictr  r   r   r<   rW   r   )rn   r   rf   ro   r   s        rF   (test_merge_asof_with_various_npartitionsrs    s    	dr~j99:Q>IIINNN	O	OB}"B4888HQ{{ $ $nR[999Cst<<<(F####	$ $rH   r   r   r   c                .   t          j        g dt          d          dg d          }t          j        |g d          }t          j        dg dig d          }t          j        |g d	          }t          j        ||gd
| d          }t          j                    5  t          j        dt                     t          ||g|           }t          ||           t          t          ||g|           j                  t          t          ||g|           j                  k    sJ t          t          ||gd          j                  t          t          ||gd          j                  k    sJ 	 d d d            d S # 1 swxY w Y   d S )Nru   rw   r'   r1   rv   r(   )r+   r,   r.   r/   r0   r6   rx   rz   r   F)axisr   rh   ignorer   r   r   )rR   r=   r>   r<   rS   r   warningscatch_warningssimplefilterFutureWarningr   r   r   r   )r   rX   r@   rY   rA   r   r   s          rF   test_indexed_concatr|    s   
   tH~~66>P>P>P	 	 	A 	q)))$$A
c3334<N<N<NOOOA
q,,,''Ay!Qad???H		 	"	" 	
 	
h666*Aq6===&(###/ATBBBGHHF%q!f4888=M
 M
 
 
 
 
 /AWEEEJKKv%q!f7;;;@P
 P
 
 
 
 
 
	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
s   "CF

FFc                (   t          j        g dt          d          dg d          }t          j        |d          }t          j        g dt          d          dg d          }t          j        |d          }t          j        g dt          d          dg d          }t          j        |d          }dd	i}||||f||||ffD ]C\  }}	}
}t          j        |
|gfd
| i|}t          j        ||	gfd
| i|}t          ||           Dd S Nru   rw   r'   r1   r!   ry   r   r+   r  r   r   r(   r   rh   Fr   )rR   r=   r>   r<   rW   r   r   r   pdf1r   pdf2rp   pdf3r   kwargsdd1dd2pd1pd2r   r   s                 rF   test_concatr    s{   <   tH~~66>P>P>P  D >$""D<   tH~~66>T>T>T  D >$""D <   tH~~66>T>T>T  D >$""De_F 
tT4 	tT4  $ $S#s 9c3Z==d=f==C:;;D;F;;&(####$ $rH   c                x   t          j        g dt          d          dg d          }t          j        |d          }t          j        g dt          d          dg d          }t          j        |d          }t          j        g dt          d          dg d          }t          j        |d          }dd	i}|j        |j        |j        |j        f|j        |j        |j        |j        ffD ]C\  }}	}
}t          j        |
|gfd
| i|}t          j        ||	gfd
| i|}t          ||           Dd S r~  )	rR   r=   r>   r<   rW   r(   r   r   r   r  s                 rF   test_concat_seriesr    s   <   tH~~66>P>P>P  D >$""D<   tH~~66>T>T>T  D >$""D <   tH~~66>T>T>T  D >$""De_F 
(	( $ $S#s 9c3Z==d=f==C:;;D;F;;&(####$ $rH   c                 v   t          j        g dt          d          dg d          } t          j        | d          }t          j        dt          d          ig d          }t          j        |d          }t          t          |d                             |d<   |                    d	
          j        	                                d	z
  |d<   ddi}t          j
        | |gfi |}t          j
        ||gfi |}t          |j        d                                          rJ t          ||           d S )Nru   rw   r'   r1   r!   r)   r  r(   r    )partition_countrh   F)rR   r=   r>   r<   rW   r   r?   assignr  cumsumr   r   r   r   r   )r  r   r  rp   r  r   r   s          rF   &test_concat_with_operation_remains_hlgr  :  sE   <   tH~~66>P>P>P  D >$""D<d8nn-5K5K5KLLLD>$""D c$s)nn%%DIA..>EEGG!KDIe_Fy$0000HYd|..v..F %V[!44DDFFFFFfhrH   c                    t          j        dg did          } t          j        g d          }t          j        | |g          }t          j        | d          }t          j        |d          }t          j        ||g          }t          ||           t          j        g dgd          }t          j        | |g          }t          j        |d          }t          j        ||g          }	|	j        j                            d          |	j        _        t          ||	d	
           t          t          j        ||g          g          t          j        | | g          g                     d S )Nr@   )r7      ,  int64dtyper    re   r(   )r   r  F)check_dtype)	rR   r=   r   r<   rW   r   _metar2   astype)
rn   r_  	df_concatro   	empty_ddf
ddf_concatempty_df_with_coldf_concat_with_colempty_ddf_with_colddf_concat_with_cols
             rF   test_concat_dataframe_emptyr  Q  s`   	sOOO,G	<	<	<B|Bg...H	2x.))I
.
+
+
+CxQ777IC+,,Ji$$$R#gFFFB(9#:;;(9qIII)S*<$=>>
 ':&?&E&L&LW&U&U# "55IIIIbic"g''B2<)@)@AAAAArH   zcatch_warnings seems flaky)reasonstrictzvalue_1, value_2)      ?r    )r  oner  z
1970-01-01)r    r  r    r  c                   t          j        d| gi          }t          j        d|gi          }t          j        ||gd          }t          |d         j                  rt                      n|d         j        }t          j        |d          }t          j        |d          }t          j        ||gd          }t          |	                    d           
                                          }	|	||gk    sJ d S )Nr(   r   ru  r    re   c                    | j         S r  )dtypesr4  s    rF   r5  z.test_concat_different_dtypes.<locals>.<lambda>  s    AH rH   )rR   r=   r   r   r  r   r<   rW   r>   map_partitionsr   )
value_1value_2df_1df_2rn   expected_dtypeddf_1ddf_2ro   dask_dtypess
             rF   test_concat_different_dtypesr  j  s      <wi())D<wi())D	D$<a	(	(	(B .bgm<<O"S'-  N4Q///EN4Q///E
)UEN
+
+
+Cs))*<*<==EEGGHHK>>:::::::rH   )r   r   r{   r   on_indexc           	        t          j        t          t          j        d                              t                              dz  t          t          j        d                    dz  d          }t          j        t          j        d          t          j        d          d          }t          j        |d          }t          j        |d          }dg}|x}}|r,|	                    d          }|	                    d          }d }t          j        d          5 }	t          j        d           t          j        |||| ||          }
t          d	 |	D                       }d d d            n# 1 swxY w Y   t          |
t           j                  r|
n|
                                }
|
                                j                                        }|r|s|rJ d S d S )
Nr$   r!   rX   rY   rX   Trecordalways)r   r}   r   r   c              3  8   K   | ]}d t          |          v V  dS )z!merge column data type mismatchesN)str)r  rs     rF   	<genexpr>z,test_merge_columns_dtypes.<locals>.<genexpr>  s.      SSq8CFFBSSSSSSrH   )rR   r=   r>   nparanger  floatr<   rW   r\  rx  ry  rz  r   anyr;   r   isnar   )r}   r  r   r   r@   rA   r   r   r   r  r   warnedhas_nanss                rF   test_merge_columns_dtypesr    s    ,29Q<<&&u--..2bill9K9Ka9OPP C ,RYq\\	!==
>
>C
sAA
sAA
B''J KKKK		 	-	-	- Th'''qRSZ[
 
 
 SSFSSSSST T T T T T T T T T T T T T T "&",77MVVV^^=M=MF{{}}#''))H000000000s   #AE77E;>E;c                   t          j        g dg dd          }t          j        |g d          }t          j        g dg dd          }t          j        |g d          }t	          t          j        ||d	d	| |
          t          j        ||d	d	|                      t          j        ||d|           }t          |t          j        ||d|                      t          d |j        D                       sJ t          t          j        ||dd| |          t          j        ||dd|                      t          t          j        ||dd| d|          t          j        ||dd| d                     t          t          j        ||| |          t          j        |||                      t          t          j        ||| |          t          j        |||                      t          t          j        ||| |          t          j        |||                      t          t          j        ||| |          t          j        |||                      t          t          j        ||d	d	| |
          t          j        ||d	d	|                      t          t          j        ||d	d	| d|          t          j        ||d	d	| d                     t          t          j        ||dd	| |          t          j        ||dd	|                      t          t          j        ||dd	| d|          t          j        ||dd	| d                     d S )Nr   r   r'   r   r   r   r   r   T)r   r   r}   r   )r   r   r}   r)   r   r}   c              3     K   | ]}|d u V  	d S r  r  r  s     rF   r  ztest_merge.<locals>.<genexpr>  s&      33QqDy333333rH   r(   r   )r   r   r}   r   )r   r   r}   )12)r   r   r}   suffixesr   )r   r   r}   r  r}   r   r|   )r   r   r}   r  r   )r   r   r}   r  )r   r   r}   r   r   r   r}   )r   r   r}   r  r   )r   r   r}   r  )	rR   r=   r<   rS   r   r   r   allrE   )r}   r   rX   r@   rY   rA   r   s          rF   
test_merger    s   
---4F4F4FGGHHA
q)))$$A
---4F4F4FGGHHA
q)))$$A
qTtn	
 	
 	
 	A$DcBBB	   Xas,,,FFBHQcs33344433&"233333333
AsSc>RRR
AsSc:::   
"	
 	
 	
 	AsScJOOO   BHQsN;;;RXaPS=T=T=TUUUBHQsN;;;RXaPS=T=T=TUUUBHQsN;;;RXaPS=T=T=TUUUBHQsN;;;RXaPS=T=T=TUUU
qTtn	
 	
 	
 	A$DcBBB	   
"	
 	
 	
 	A$DcJWWW   
As#~VVV
As#>>>   
"	
 	
 	
 	As#
SSS    rH   r   c                   t          j        g dg dd          }t          j        g dg dd          }t          j        |d          }t          j        |d          }|                    |d|	          }|                    |d|	          }t          ||d
           |                    d |j                                                   d S )N)r    r    r!   r!   r$   r%   rJ   ry   )r@   val)r   r   r"   r"   )r  r   r   r   r#   re   r@   r  Fr   c                    | S r  r  r4  s    rF   r5  z*test_merge_empty_left_df.<locals>.<lambda>  s    A rH   )meta)	rR   r=   r<   rW   r   r   r  r  r   )r   r}   r{   r   dd_leftdd_rightmergedr   s           rF   test_merge_empty_left_dfr    s     <lll<<<@@AADL|||4D4D4DEEFFEnTq111G~e333H]]8]55Fzz%CSz11HfhE2222 ++FL99AACCCCCrH   c                    t          j        g dg dd          } t          j        g dg dd          }t          j        t          d          5  t          j        | |d	           d d d            d S # 1 swxY w Y   d S )
N)r   r   r   r   r   r  r   )D0D1D2D3z1dask.dataframe.merge does not support how='cross'r   crossr|   )rR   r=   rj   rk   rl   r<   r   )r{   r   s     rF   test_merge_how_raisesr    s    <))))))	
 	
 D L))))))	
 	
 E 
M
 
 
 + + 	u'****+ + + + + + + + + + + + + + + + + +s   A77A;>A;rb   ))r"   r"   )r"   r    )r    r"   leftsemileftantienginepandasz,Pandas does not support leftsemi or leftanti)r  )markscudfc                   | dk    r(t          j        d          }t          j        d          }t          j        t	          j        dd          g dg dg dd	          }t          j        g d
g dg dd          }| dk    re|                    |          }|                    |          }|                    ||d                   }|                    ||d                   }n8t          j        ||d                   }t          j        ||d                   }|	                    |d|          
                    dg          }	|	                    |d|          
                    dg          }
t          |
|	d           d S )Nr  	dask_cudfe   j   )stop)JohnTomHarryRahulSakil)CalMumDelBanr  )iP  i@  i8 i`  i_ )emp_idnamecitysalary)i  i  i  i  i  )g   r  i   f   r  )DaskSparkr   PythonR)skill_idr  
skill_namer   re   r    r  r  Fr   )rj   importorskiprR   r=   r  r  rW   	from_cudfr<   r   r   r   )r  r}   rb   r  r  empskillsdd_emp	dd_skillsexpectr   s              rF   test_merge_tasks_semi_anti_cudfr	  +  s     "6**'44	
,i#...>>>777999		
 	
 C \111///???	
 	
 F s##!!&))$$SeAh$??''E!H'EE		q:::N6uQx@@@	YYv(Y44@@(LLF\\)c\::FFzRRFff%000000rH   c                 <   t          j        g dg dd          } t          j        g dg dd          }t          j        | d          }t          j        |d          }|                    |dd	d
          }t          d |j        D                       rJ d S )NrI   rK   rL   rM   r"   re   r!   r@   rO   tasks)r   r   r   c              3  *   K   | ]}d |d         v V  dS )partdr   Nr  r  ks     rF   r  z2test_merge_tasks_passes_through.<locals>.<genexpr>h  s*      44q7ad?444444rH   )rR   r=   r<   rW   r   r  r   )r@   rA   rB   rC   ccs        rF   test_merge_tasks_passes_throughr  _  s    
0007L7L7LMMNNA
0007L7L7LMMNNA	q	)	)	)B	q	)	)	)B	"cC	A	AB44BG4444444444rH   c                N   t          j        g dg dd          }t          j        g dg dd          }t          j        g dg ddt          d                    }t          j        g dg ddt          d                    }|}t          j        g dg ddt          d	                    }|}t          j        g d
g ddt          d	                    }	t          j        g dg ddt          d                    }
t          j        g dg ddt          d                    }t          j        g dg ddt          d                    }t          j        g dg ddt          d                    }t          j        g dg ddt          d                    }t          j        g dg ddt          d                    }d }||f||f||f||	f|
|f||f||ffD ]\  }}dD ]\  }}t          j        ||          }t          j        ||          }t          t          j        ||| dd|           |t          j        ||| dd          |j        j                             t          t          j        ||| dd|           |t          j        ||| dd          |j        j                             t          t          j        ||| dd|d           |t          j        ||| ddd          |j        j                             t          t          j        ||| dd|d           |t          j        ||| ddd          |j        j                             t          |                    || dd|           ||                    || dd          |j        j                             t          |                    || dd|           ||                    || dd          |j        j                             t          t          j        ||| dd|          t          j        ||| dd                     t          t          j        ||| dd|          t          j        ||| dd                     t          t          j        ||| dd|d           t          j        ||| ddd!                     t          t          j        ||| dd|d           t          j        ||| ddd!                     t          t          j        ||| dd|          t          j        ||| dd                     t          t          j        ||| dd|          t          j        ||| dd                     t          |                    || dd|          |                    || dd                     t          |                    || dd|          |                    || dd                     t          |                    || dd|          |                    || dd                     t          |                    || dd|          |                    || dd                     d S )"NrI   rK   rL   rM   rP   r1   )r%   rJ   ry   r   r$   r#   r"   r!   abdg)r   r+   r  r   )r    r    r!   r!   r"   r"   r#   lmnopqr)r    r    r    r    abcdcdefghir    r!   r    r!   r  fghic                    t          |           dk    r-|                     | j                            |                    S | S Nr   r?   r\  r2   r  outr  s     rF   	fix_indexz/test_merge_by_index_patterns.<locals>.fix_index  :     s88q====!1!1%!8!8999
rH   )r!   r!   )r"   r!   )r!   r"   Tr}   r   r   r   r}   r   r   )r}   r   r   r   	indicator)r}   r   r   r$  r@   rN   r}   r   r   r   r}   r   r   rA   rO   )r}   r   r   r   r$  )r}   r   r   r$  )
rR   r=   r>   r<   rW   r   r   r2   r  r   )r}   r   pdf1lpdf1rpdf2lpdf2rpdf3lpdf3rpdf4lpdf4rpdf5lpdf5rpdf6lpdf6rpdf7lpdf7rr  pdlpdrlpartrpartddlddrs                          rF   test_merge_by_index_patternsr;  k  s    L444;P;P;PQQRREL444;P;P;PQQRREL###*?*?*?@@Y  E L###*?*?*?@@Y  E EL|||,,,??tF||TTTEEL\\\BB$v,,WWWE L###*?*?*?@@Y  E L|||,,,??tF||TTTEL###*?*?*?@@Y  E L|||,,,??tF||TTTEL###*?*?*?@@Y  E L|||,,,??tF||TTTE   
						 B BS
 y	 y	LE5
 .e,,C.e,,C# $*   	HS#34TRRRIO    # $*   	HS#34TRRRIO    # $*"   	H#'$("&   IO
 
  , # $*"   	H#'$("&   IO
 
  . 		# $*    	IIcstINNIO    		# $*    	IIcstINNIO    #sS.   csC#FFF	   #sS.   csC#FFF	    *"   csC#QUVVV    *"   csC#QUVVV   #sS.   csC#FFF	   #sS.   csC#FFF	   		S#^    		#3c	BB	   		S#^    		#3c	BB	   		S#^    		#3c	BB	   		S#^    		#3c	BB	   iy	B BrH   zbreaks with pandas=2.1.0+c                *   d }t          j        t          d          g ddt          d                    }t          j        t          d          g ddt          d                    }|}t          j        t          d          g ddt          d                    }|}t          j        t          d          g d	dt          d                    }t          j        t          d
          g ddt          d                    }	t          j        t          d          g d	dt          d                    }
t          j        t          d          g ddt          d                    }t          j        t          d          g d	dt          d                    }t          j        t          d          g ddt          d                    }t          j        t          d          g d	dt          d                    }t          j        t          d          g ddt          d                    }t          j        t          d          g d	dt          d                    }||f||f||f|	|
f||f||f||ffD ]\  }}dD ]\  }}t          j        ||          }t          j        ||          }t          |                    || |           ||                    ||           |j        j                             t          |                    || |           ||                    ||           |j        j                             t          |                    || dd|           ||                    || dd          |j        j                             t          |                    || dd|           ||                    || dd          |j        j                             t          |                    || ddd          |                    || ddd                     t          |                    || ddd          |                    || ddd                     t          |
                    || dd          |
                    || dd                     t          |
                    || dd          |
                    || dd                     t          |
                    || dd          |
                    || dd                     t          |
                    || dd          |
                    || dd                     d S ) Nc                    t          |           dk    r-|                     | j                            |                    S | S r  r  r  s     rF   r  z.test_join_by_index_patterns.<locals>.fix_index^  r   rH   rP   rK   rL   r1   rM   gfedcbar  r  abcabcer4   r  r  r  ababaabbccdaabbr  r!  r  r|   lr  )r}   r   r   r   r   r@   )r}   r   r   r   rN   T)r}   r   r   )r}   r   r   )rR   r=   r>   r<   rW   r   r   r2   r  r   r   )r}   r   r  r'  r(  r)  r*  r+  r,  r-  r.  r/  r0  r1  r2  r3  r4  r5  r6  r7  r8  r9  r:  s                          rF   test_join_by_index_patternsrD  Z  s      L9oo$9$9$9::$y//  E L9oo$9$9$9::$y//  E EL9oo$9$9$9::$y//  E ELtF||,,,??tF||TTTEL9oo$9$9$9::$y//  E LtF||,,,??tF||TTTE L9oo$9$9$9::$y//  E LtF||,,,??tF||TTTEL9oo$9$9$9::$y//  E LtF||,,,??tF||TTTEL9oo$9$9$9::$y//  E LtF||,,,??tF||TTTE 
						 E ES 5 <	 <	LE5.e,,C.e,,C#~>>	#((3C(00#)/BB   #~>>	#((3C(00#)/BB  
 S#sN    	HHSc3HDDcio 	   S#sN    	HHSc3HDDcio 	   ##sCHH##sCHH  
 ##sCHH##sCHH   		#3	FF		#3	FF   		#3	FF		#3	FF   		#34#	FF		#34#	FF   		#34#	FF		#34#	FF   s<	E ErH   c                 F   t          j        dg dig d          } t          j        | d          }t          j        dg dig d          }|                     |d	d
          }|                    |d	d
          }|j        |j        k    sJ t          ||           d S )Nr@   r   )r   r    r!   r1   r    re   rA   )r  g       @g      @r   )r}   r   )rR   r=   r<   rW   r   rE   r   )rn   ro   right_dfr   actuals        rF    test_join_gives_proper_divisionsrH    s    	sOOO,III	>	>	>B
.
+
+
+C|S///2///JJJHwwxWw55HXXhGX44Fs},,,,hrH   c                   d }t          j        t          d          t          d          g ddt          d                    }t          j        t          d          t          d          g ddt          d                    }t          j        t          d          t          d	          g ddt          d                    }t          j        t          d
          t          d          g ddt          d                    }t          j        t          d          t          d          g ddt          d                    }t          j        t          d          t          d          g ddt          d                    }||f||f||ffD ]\  }	}
dD ]\  }}t          j        |	|          }t          j        |
|          }t          |                    || |           ||	                    |
|           |	j        j                             t          |                    || |           ||
                    |	|           |
j        j                             t          t          j	        ||| dd|           |t          j	        |	|
| dd          |	j        j                             t          t          j	        ||| dd|           |t          j	        |
|	| dd          |
j        j                             t          t          j	        ||| dd|          t          j	        |	|
| dd                     t          t          j	        ||| dd|          t          j	        |	|
| dd                     t          t          j	        ||| dd|          t          j	        |
|	| dd                     t          t          j	        ||| dd|          t          j	        |
|	| dd                     t          t          j	        ||| ddgddg|          t          j	        |	|
| ddgddg                     d S )Nc                    t          |           dk    r-|                     | j                            |                    S | S r  r  r  s     rF   r  z1test_merge_by_multiple_columns.<locals>.fix_index  s8    s88q====!1!1%!8!8999
rH   
abcdefghij)
r    r!   r"   r#   r$   r%   rJ   ry   r   r+   r   r1   )
r+   r   ry   rJ   r%   r$   r#   r"   r!   r    )rO   ef
abcdeabcde
abcabcabca
edcbaedcba
aaabbbcccd
fghijklmno
aaaaaaaaaa
aaabbbccaa
abbbbbbbbb
ABCDEFGHIJr!  r  r|   Tr"  r#  r@   rO   r%  r&  rA   rL  )rR   r=   r>   r<   rW   r   r   r2   r  r   r   )r}   r   r  r'  r(  r)  r*  r+  r,  r5  r6  r7  r8  r9  r:  s                  rF   test_merge_by_multiple_columnsrW    sh      Ll##l##000	
 	

 <    E Ll##l##000	
 	

 <    E Ll##l##000	
 	

 <    E Ll##l##000	
 	

 <    E Ll##l##000	
 	

 <    E Ll##l##000	
 	

 <    E U^eU^eU^D P PS4 O	 O	LE5.e,,C.e,,C#~>>	#((3C(00#)/BB   #~>>	#((3C(00#)/BB  
 # $*   	HS#34TRRRIO    # $*   	HS#34TRRRIO     #sS.   csC#FFF	   #sS.   csC#FFF	   #sS.   csC#FFF	   #sS.   csC#FFF	    #J!3Z*   csS#J#sTTT
 
 
 
KO	P PrH   r  int)id_vars)
value_varsobjmyvar)rZ  var_names1myval)rY  rZ  
value_names2c                   t          j        t          d          dz  t          d          dz  t          d          dz  t          j                            d          d          }t          r|                    d	d	d
          }t          j	        |d          }t          t          j        |fi | t          j        |fi |            t           |j        di |  |j        di |            d S )Nr  r$   XYr+   abcder#   r,   )r[  r^  ra  rX  zstring[pyarrow])r^  ra  r  )rR   r=   r>   r  randomrandnpar  r<   rW   r   melt)r  pdfro   s      rF   	test_meltrj  r  s     ,<<!#t**r/w--!#9??2&&		
 	
 C 
 M jj 19JKKLL
.a
 
 CBGC""6""BGC$:$:6$:$:;;;HCHv 2 26 2 233333rH   c                    t          j        g dt          d          dg d          } t          j        | d          }t          j        g dt          d	          d
          }|                    |dd          }t          |j        d                                          rJ t          d |j        D                       sJ t          |                    |dd          |                     |dd                     d S )Nr   r&   r'   r*   r1   r"   re   r3   r4   r  r(   r   r  r~   c              3  *   K   | ]}d |d         vV  dS r   r   Nr  r  s     rF   r  z<test_cheap_inner_merge_with_pandas_object.<locals>.<genexpr>  +      66y!$666666rH   )rR   r=   r>   r<   rW   r   r   r   r   r  r   )r@   darA   dcs       rF   )test_cheap_inner_merge_with_pandas_objectrq    s	   
   tH~~66>V>V>V	 	 	A 
q	)	)	)B
<<<d6ll;;<<A	!	)	)B$RWb11AACCCCC66bg66666666BHHQ3GH,,aggaCWg.M.MNNNNNrH   flipc           	        t          j        g dt          d          dg d          }t          j        |d          }t          j        g dt          d	          d
          }t          j        |dd          }| r||fn||f}| r||fn||f}t          j        |ddd}t          |j        d                                          rJ t          d |j        D                       sJ |j        j
                                        |j        j
                                        z  }|j        j
                                        }	t          |	|z
            dk    sJ t          |t          j        |ddd           d S )Nr   r&   r'   r*   r1   r"   re   r3   r4   r  r    Frg   r(   r   r  r~   c              3  *   K   | ]}d |d         vV  dS rm  r  r  s     rF   r  z4test_cheap_single_partition_merge.<locals>.<genexpr>  rn  rH   )rR   r=   r>   r<   rW   r   r   r   r   r  layerskeysr?   r   )
rr  r@   rB   rA   rC   	pd_inputsinputsr  input_layersoutput_layerss
             rF   !test_cheap_single_partition_merger{    s   
   tH~~66>V>V>V	 	 	A 
q	)	)	)B
<<<d6ll;;<<A	qu	5	5	5B*AQFI+b"XXB8F	6cw	/	/	/B$RWb11AACCCCC66bg66666666 7>&&((27>+>+>+@+@@LGN''))M}|+,,1111B)999:::::rH   c                    t          j        g dt          d          dg d          } t          j        | d          }t          j        g dt          d	          d
          }t          j        |dd          }|                    |dd          }t          |j        d                                          rJ |j	        rJ t          |           |                    |dd          }|j	        rJ t          |           d S )Nr   r&   r'   r*   r1   r"   re   r3   r4   r  r    Frg   r(   r   r  r~   )rR   r=   r>   r<   rW   r   r   r   r   ri   r   )r@   rB   rA   rC   rG  s        rF   +test_cheap_single_partition_merge_divisionsr}    s   
   tH~~66>V>V>V	 	 	A 
q	)	)	)B
<<<d6ll;;<<A	qu	5	5	5BXXbSgX..F$V["55EEGGGGG%%%%VXXbSgX..F%%%%VrH   r{   c                   t          j        t          d          t          d          dt          d                    }t	          j        |d          }t          j        g dt          d          dt          d	                    }t	          j        |d          }|r||fn||f}|r||fn||f}t	          j        |d
d| d}t          j        |d
d| d}	t          |j        d          	                                rJ t          ||	           t	          j        |dd
| d}t          j        |dd
| d}	t          |j        d          	                                rJ t          ||	           d S )Nry   ababbddar  r1   r    re   r3   r4   r#   Tr(   r   r   r}   r~   r  )rR   r=   r   r>   r<   rW   r   r   r   r   r   )
r}   rr  r@   rB   rA   rC   rw  rx  rG  r   s
             rF   +test_cheap_single_parition_merge_left_rightr    s    	588$z*:*:;;588LLLA	q	)	)	)B
<<<d6ll;;588LLLA	q	)	)	)B*AQFI+b"XXB8FXv$#FFFFxtcsKKKH$V["55EEGGGGGfhXvs#FFFFxCTsKKKH$V["55EEGGGGGfhrH   c                 L   t          j        g dt          d          dg d          } t          j        | d          }t          j        g dt          d	          d
          }t          j        |dd          }|                    |ddd          }|                     |ddd          }|j                            d          |_        t          |j	        d          
                                rJ |j        rJ t          ||           |                    |ddd          }|                    | ddd          }|j                            d          |_        t          |j	        d          
                                rJ |j        rJ t          ||           d S )Nr   r&   r'   r*   r1   r"   re   r3   r4   r  r    Frg   Tr(   r   r  r  r~   )r   r   r}   )rR   r=   r>   r<   rW   r   r2   r  r   r   r   ri   r   )r@   rB   rA   rC   rG  r   s         rF   *test_cheap_single_partition_merge_on_indexr    s   
   tH~~66>V>V>V	 	 	A 
q	)	)	)B
<<<d6ll;;<<A	qu	5	5	5BXXbTCWXEEFwwqTCWwEEH
 ^**733HN$V["55EEGGGGG%%%%fhXXbdCWXEEFwwqdCWwEEH^**733HN$V["55EEGGGGG%%%%fhrH   c                    t          j        g dt          d          dddt          d                    } t          j        dd	gd
dt          d          dt          d                    }t          j        | d          }t          j        ||d                                          }t          |j                  dk    sJ d S )Nr  abcrq  r  )rX   rY   r   DDCBAr/  r#   r$   g      @barab)GHIrY   GHIBr    re   rY   r   )r  r   rY   rX   r  r  r  )	rR   r=   r>   r<   rW   r   r   tupler   )r]   r^   ro   r  s       rF   test_merge_maintains_columnsr    s    
,iid5kkC@@$v,,  C ,!f3Ud<<d6ll  C .!
,
,
,CXc33'''//11F  $GGGGGGGrH   c                N   t          j        dg dig d          }t          j        dg dig d          }t          j        |dd          }t          j        |d	
          }|                    |d|           }|                    |d          }t          ||           d S )Nr(   r    r!   r"   r#   r$   r1   r)   r$   r#   r"   r!   r    r"   Frg   r!   re   r   r  r|   )rR   r=   r<   rW   r   r   )r   r@   rA   rB   rC   r   r   s          rF   "test_merge_index_without_divisionsr    s    
c???+???CCCA
c???+???CCCA	qu	5	5	5B	q	)	)	)BWWRWnW==FvvaWv%%HfhrH   c                 *   t          j        dt          j                            dd          i          } t          j        dt          j                            dd          it          j                            dd                    }t          j        | d          }t          j        |d          }t          j        | |d	d
          }t          j        ||d	dd          }t          ||           t          |j
                  dk     sJ d S )Nr(   r7   i  sizer)   r1   re   r!   Tr   r   r  )r   r   r   i  )rR   r=   r  re  randintr<   rW   r   r   r?   r   )r@   rA   rB   rC   rN   r  s         rF   *test_half_indexed_dataframe_avoids_shuffler    s    
c29,,St,<<=>>A
	bi#../ry7H7HSV7H7W7W	 	 	A 
s	+	+	+B	q	)	)	)B
A$555A	"bTC	I	I	IBArNNNrw<<#rH   c                    t          j        dg dig d          } t          j        dg dig d          }t          j        | dd          }t          j        |d	
          }t	          j        t                    5  t          j        ||d|j                   d d d            n# 1 swxY w Y   t	          j        t                    5  t          j        |||j	        |j                   d d d            d S # 1 swxY w Y   d S )Nr(   r  r1   r)   r  r"   Frg   r!   re   rk  )
rR   r=   r<   rW   rj   rk   NotImplementedErrorr   r)   r(   r@   rA   rB   rC   s       rF   &test_errors_for_merge_on_frame_columnsr  /  s   
c???+???CCCA
c???+???CCCA	qu	5	5	5B	q	)	)	)B	*	+	+ 5 5
Rrt44445 5 5 5 5 5 5 5 5 5 5 5 5 5 5 
*	+	+ 6 6
R55556 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6s$   ?B))B-0B-#C==DDc                 ,   t          j        g d          } t          j        | dd          }t          j        |gd          }t          |t          j                  sJ t          j        |gd          }t          |t          j                  sJ d S )Nr3   r!   Frg   r   r  r    )rR   Seriesr<   rW   r   r;   r=   )r@   rB   rN   s      rF   test_concat_one_seriesr  =  s    
	,,,A	qu	5	5	5B
	2$QAa#####
	2$QAa&&&&&&&rH   c                    t          j        g d          } t          j        g d          }t          j        | dd          }t          j        |dd          }|j        rJ t          j        t                    5  t          t          j	        | |gd          t          j	        ||gd                     d d d            n# 1 swxY w Y   t          j        |dd          }t          j
        t                    5  t          j	        ||gd           d d d            n# 1 swxY w Y   t          j        d	          5 }t          j	        ||gdd
           d d d            n# 1 swxY w Y   |rJ d S )Nr3   r#   r"   r!   r    r!   Frg   r    r  Tr  )ru  ignore_unknown_divisions)rR   r  r<   rW   ri   rj   warnsUserWarningr   r   rk   rl   rx  ry  )r@   rA   rB   rC   r  r  s         rF   test_concat_unknown_divisionsr  H  sJ   
	,,,A
	,,,A	qu	5	5	5B	qu	5	5	5B!!!!	k	"	" J J")QF+++RYBxa-H-H-HIIIJ J J J J J J J J J J J J J J 
qu	5	5	5B	z	"	" $ $
	2r(####$ $ $ $ $ $ $ $ $ $ $ $ $ $ $ 
	 	-	-	- C
	2r(TBBBBC C C C C C C C C C C C C C Cs6   ==CC
C
D&&D*-D*E,,E03E0c                    t          j        g d          } t          j        g d          }t          j        | dd          }t          j        |dd          }t	          j        t                    5  t	          j        t                    5  t          j	        ||gd          
                                 d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nr   r  r!   Frg   r    r  )rR   r  r<   rW   rj   rk   rl   r  r  r   r   r  s       rF   $test_concat_unknown_divisions_errorsr  \  sU   
	$$$%%A
	,,,A	qu	5	5	5B	qu	5	5	5B	z	"	" 2 2\+&& 	2 	2Ir2hQ'''//111	2 	2 	2 	2 	2 	2 	2 	2 	2 	2 	2 	2 	2 	2 	22 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2s6   4C+C9CC			CC		CC!$C!c                 Z   t          j        g dg dd          t          j        g dg dd          t          j        g dg dd          d} t          dddt          j                    	          }t          j        | d
|d d g          }t          j        g dg dd          t          j        g dg dd          t          j        g dg dd          d} t          j        | d|d d g          }t          j        g dg dd          t          j        g dg dd          d} t          dddt          j                    	          }t          j        | d|d d g          }t          j        g dg dg dd          t          j        g dg dg ddg d          d} t          ddddt          j        g d          t          j                              }t          j        | d|g d          }||g||g||gg}t          j        |g          |u sJ |D ]}d |D             }t          j        |d          }	t          j        |          }
|
j        |d         j        |d         j        z   k    sJ |
j        d|
j        dz   z  k    sJ t          |	|
           t          |
j
                  t          t          j        |          j
                  k    sJ t          j        |dd           }	t          j        |d!          }
|
j        |d         j        |d         j        z   k    sJ |
j        d|
j        dz   z  k    sJ t          |
|
           t          |
j
                  t          t          j        |d!          j
                  k    sJ d S )"Nr  r#   r$   r%   rL   r"   r!   r    rJ   ry   r   r   r   r   )r(   r   )r(   r    )r(   r!   r9   parent_metar(   r  r.   r/   r0   r-   r,   r+   r5   r6   Z   r)   r   r)   r    )r)   r!   r)   rA   rN   r  r  )rA   rN   rO   )r  r6   r5   )r"   r#   r$   r1   )r2   r  )r   r"   r$   c                6    g | ]}|                                 S r  r   )r  _cs     rF   r	  z test_concat2.<locals>.<listcomp>  s     ...2"**,,...rH   Frh   r   r    r  r   r   rh   rw  )rR   r=   r   r<   Indexr   rf   rE   r   setr   )dskr  r@   rA   rN   rO   casescasepdcaser   r   s              rF   test_concat2r  g  s&   ,YYYYYY??@@,YYYYYY??@@,YYYYYY??@@ C
 4d++HHHD
S#tdD\22A,\\\EEFF,\\\EEFF,\\\			BBCC C
 	S#tdD\22A ,\\\EEFF,\\\EEFF C 4d++HHHD
S#tdD\22A ,,,\\\EE
 
 ,,,\\\EEYYY
 
 
	 C D))hr4  LNN  D
 	S#tYYY//AVaVaV$E9aS>>Q K K.....9V%0004!T!W%847;N%NNNNN7f.@1.D#EEEEE(F###6;3ry';#<#<<<<<9V'>>>4g...!T!W%847;N%NNNNN7f.@1.D#EEEEE&&!!!6;3ryG'D'D'D'I#J#JJJJJJK KrH   c                 f   t          j        t          j                            dd          t          d          t          d                    } t          j        t          j                            dd          t          d          t          d                    }t          j        t          j                            dd          t          d          t          d	                    }t          j        | d
          }t          j        |d          }t          j        |d
          }t          j        | |gd          }t          j        ||g          }|j	        |j	        d d         |j	        z   k    sJ |j
        |j
        |j
        z   k    sJ t          ||           t          t          j        ||gd          t          j        | |g                     t          j        | ||gd          }t          j        |||g          }|j	        |j	        d d         |j	        d d         z   |j	        z   k    sJ |j
        |j
        |j
        z   |j
        z   k    sJ t          ||           t          t          j        |||gd          t          j        | ||g                     d S )Nr%   r$   ABCDErw   r   r2   ABCFGghijklABCHImnopqrr!   r"   Fr  r~   Tinterleave_partitions)rR   r=   r  re  rf  r>   r<   rW   r   rE   rf   r   )r  r  r  r   rp   r   r   r   s           rF   test_concat3r    s   <
	1tG}}DNN  D <
	1tG}}DNN  D <
	1tG}}DNN  D >$""D>$""D>$""Dy$E222HYd|$$Ft~crc2T^CCCCC!1D4D!DDDDDfh
	4,d;;;RYd|=T=T   y$d+%888HYdD)**FssdnSbS11DNB    4++d.>>    fh
	4t$DAAA
	4t$%%    rH   c            	     ^   t          j        t          j                            dd          t          d          t          d                    } t          j        t          j                            dd          t          d          t          d                    }t          j        t          j                            dd          t          d	          t          d                    }t          j        | d
          }t          j        |d          }t          j        |d
          }d}||g||g||g||g||g||g||gg}|D ]}d |D             }	t          t          j	        |d          t          j	        |	d                     t          t          j	        |dd          t          j	        |	dd                     d}t          j        t                    5 }
t          j	        ||gdd           d d d            n# 1 swxY w Y   |t          |
j                  v sJ d S )Nr+   r$   r  rK  r  r   fghijklmnopqrr%   CDEXYZr!   r"   zyAll inputs have known divisions which cannot be concatenated in order. Specify interleave_partitions=True to ignore orderc                6    g | ]}|                                 S r  r  r  rN   s     rF   r	  z6test_concat4_interleave_partitions.<locals>.<listcomp>       ,,,!!))++,,,rH   Tr  Fr  r   r   r  r  z!'join' must be 'inner' or 'outer'invalid)rR   r=   r  re  rf  r>   r<   rW   r   r   rj   rk   rl   r  value)r  r  r  r   rp   r   msgr  r  r  errs              rF   "test_concat4_interleave_partitionsr    s~   <
	AWT,=O=O  D <
	AWT/=R=R  D <
	AXd?>S>S  D >$""D>$""D>$""D	5  
t	t	t	t	t	t	tE  	
 	
,,t,,,Id$7776PU9V9V9V	
 	
 	
 	IdEEEIf7777	
 	
 	
 	

 .C	z	"	" Lc
	4,YdKKKKL L L L L L L L L L L L L L L#ci..      s   $H

HHc                 :	   t          j        t          j                            dd          t          d          t          d                    } t          j        t          j                            dd          t          d          t          d                    }t          j        t          j                            dd          t          d          t          d                    }t          j        t          j                            dd          t          d	          t          d                    }t          j        t          j                            dd          t          d	          t          d
                    }t          j        | d          }t          j        |d          }t          j        |d          }t          j        |d          }t          j        |d          }	||g||g||g||	g||g||	g|	||g|	|g|j        |j        g|j	        |j	        g|j        |	j        g|j        |j	        g|j	        |j
        g|j        |	j        g||j        g|j	        |g|	|j        |gg}
|
D ]}d |D             }t          t          j        |d          t          j        |d                     t          t          j        |dd          t          j        |d                     t          t          j        |d          t          j        |d                     t          t          j        |dd          t          j        |dd                     ||g||g| |g| j        |j        g|j	        |j	        g||j        g|j	        |g|| |j	        gg}
|
D ]}d |D             }t          t          j        |d          t          j        |                     t          t          j        |dd          t          j        |d                     t          t          j        |d          t          j        |d                     t          t          j        |dd          t          j        |dd                     d S )NrJ   r$   r  rP   r  r%   FGHIJKr  FGHABfklmnopr!   r"   c                6    g | ]}|                                 S r  r  r  s     rF   r	  z test_concat5.<locals>.<listcomp>"  r  rH   Tr  Fr  r   r  rw  r    r  )ru  r   c                d    g | ]-}t          |t                    r|                                n|.S r  )r;   r   r   r  s     rF   r	  z test_concat5.<locals>.<listcomp>B  s3    LLL!Av!6!6=!))+++ALLLrH   )rR   r=   r  re  rf  r>   r<   rW   rX   Fr  rY   r   r   )r  r  r  pdf4pdf5r   rp   r   ddf4ddf5r  r  r  s                rF   test_concat5r    s}   <
	1tG}}DOO  D <
	1tH~~T)__  D <
	1tH~~T)__  D <
	1tG}}DOO  D <
	1tG}}DOO  D >$""D>$""D>$""D>$""D>$""D 
t	t	t	t	t	t	tT	t							tv		tvt#E(  
 
,,t,,,Id$777If5)))	
 	
 	

 	IdEEEIf7+++	
 	
 	

 	")Dq)))29V!+D+D+DEEEId111If17333	
 	
 	
 	
 
t	t	t			tv		tTV	E  
 
LLtLLL")D===ry?P?PQQQIdEEEIf7+++	
 	
 	

 	")Dq)))29V!+D+D+DEEEId111If17333	
 	
 	
 	

 
rH   zknown, cat_index, divisions)TTFz3.12.0z>fails on pandas dev: https://github.com/dask/dask/issues/10558)r  rk   r  )TFF)FTF)FFFc                    t          j        t          d          t          j        d          t          d          t          j        dd          d          t          j        t          d          t          j        dd          t          d	          t          j        ddd          d          t          j        t          d
          t          j        dd          t          d          t          j        ddd          d          g}|D ]@}|j                            d          |_        |j                            d          |_        Ard |D             }fd|D             } s)t          |d         j	        dgd          |d         _	         fd}dD ]} ||||          }t          |j                  sJ t          |j                   k    sJ  |d |D             d |D             |          }t          |           k    sJ r! |d |D             d |D             |          } ||d         ddg         g|dd          z   |d         ddg         g|dd          z   |          }t          |d          rt          |j                  sJ t          |j                   k    sJ d S )Nxxxxxr$   abcbcf8r  )wr(   r)   r   yyyyyr+   abbbazzzzzr   bcbcccategoryc                B    g | ]}|                     |j                  S r  )r\  r)   )r  rn   s     rF   r	  z+test_concat_categorical.<locals>.<listcomp>  s&    666",,rt$$666rH   c                >    g | ]}t          j        |d           S )r!   rg   r<   rW   )r  prE   s     rF   r	  z+test_concat_categorical.<locals>.<listcomp>  s*    PPPAr~aQY???PPPrH   r   r)   Tr1   c                `   t          ||          }t          j         | |          }t          ||           	rVt          t          j        |j        |                                          }d |D             D ]}t          |j        |           rt          |j
                  	k    sJ |S )Nrw  r  c                ,    g | ]}|j         d d         S r  )r:  r  is     rF   r	  zEtest_concat_categorical.<locals>.check_and_return.<locals>.<listcomp>  s"    000QafRaRj000rH   )r   r<   r   r   r=   r   __dask_keys__r   r  r   r2   )
ddfsdfsr   solresrb   r  	cat_indexrE   knowns
          rF   check_and_returnz1test_concat_categorical.<locals>.check_and_return  s    St$$$i4yIII#s 	),ch(9(9(;(; E 10%000 ) )39a((((H 4SY ? ?5 H H H H
rH   )r   r   c                    g | ]	}|j         
S r  r)   r  s     rF   r	  z+test_concat_categorical.<locals>.<listcomp>  s    555555rH   c                    g | ]	}|j         
S r  r  r  s     rF   r	  z+test_concat_categorical.<locals>.<listcomp>  s    7L7L7L7L7L7LrH   c                    g | ]	}|j         
S r  r4  r  s     rF   r	  z+test_concat_categorical.<locals>.<listcomp>  s    #9#9#9AAC#9#9#9rH   c                    g | ]	}|j         
S r  r4  r  s     rF   r	  z+test_concat_categorical.<locals>.<listcomp>  s    ;P;P;PAAC;P;P;PrH   r(   r    r  )rR   r=   r>   r  r  r  r  r)   r   r  r   hasattr)	r   r  rE   framesrn   dframesr  r   r  s	   ```      rF   test_concat_categoricalr
  S  sA   B 	']]Yq\\']]Yq---	 	
 	
 	']]Yq"%%']]Yq"D111	 	
 	
 	']]Yr2&&']]Yr2T222	 	
 	
#F4  ' 't{{:&&t{{:&& 766v666PPPPPPPG W1'!*2BSEQUVVV
       # 4 4w55#CE*****#CE**e3333 55W5557L7LV7L7L7LdSS#C((E1111  	X""#9#9#9#9#9;P;P;P;P;PRVWWC QZc
#$wqrr{2AYSz"#fQRRj0
 

 3$$C(<SU(C(CCCC#CE**e33333+4 4rH   c                    t          j        g dd          } t          j        ddgd          }t          j        | d          j                                                            d          }t          j        |d                              d          }t          |                     d          |                    d          g          }t          j        ||g          }t          ||           d S )Nr   r  r  r@   rA   r!   rX   )	rR   r  r<   rW   cat
as_unknownto_framer   r   )r@   rA   ro  dbr   r   s         rF   $test_concat_categorical_mixed_simpler    s    
	///444A
	3*J///A	1			!	,	,	.	.	7	7	<	<B	1			&	&s	+	+Bqzz#

3899HYBx  FfhrH   c                    t          j        ddgit          j        dgd                    } t          j        ddgit          j        dgd                    }| j                            d	          j                            ddg          | d<   |j                            d	          j                            ddg          |d<   t          j        | d
          }t          j        |d
          }t          | j
        d d         |j
        d d         g          }|j        j        dk    sJ t          j	        ||g          }t          j	        | |g          }t          ||           d S )Nr(   r@   z2015-03-24 00:00:16zdatetime64[ns]r  r1   rN   z2015-03-29 00:00:44r  r    r   zM8[ns])rR   r=   DatetimeIndexr(   r  r  set_categoriesr<   rW   r   r:  r2   r  r   )b2b3db2db3r   r   s         rF   test_concat_datetimeindexr    sf   		se 56>NOOO
 
 
B 
	se 56>NOOO
 
 
B
 dkk*%%)88#sDDBsGdkk*%%)88#sDDBsG
.Q

C
.Q

CRWRaR["'"1"+.//F<))))YSz""Fy"b""HfhrH   c                d   t           rnt          j        t          d          5  |                    |          }|                     |          }t          ||           d d d            n# 1 swxY w Y   n:|                    |          }|                     |          }t          ||           |S )Nzappend method is deprecatedr   )r   rj   r  r{  appendr   )dask_objdask_append
pandas_objpandas_appendr   r   s         rF   check_append_with_warningr    s     $\-/LMMM 	( 	(!((77H__[11Ffh'''	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	(
 $$]33--&(###Ms   ;A**A.1A.zpandas removed appendc                 P   t          j        g dg dd          } t          j        g dg ddg d          }t          j        g dg ddg d          }t          j        | d          }t          j        |d          }t          j        |d          }t          j        ddgd	d
dg          }t          ||| |           t          ||| |           t          |j        |j        | j        |j                   t          ||| |           t          |j        |j        | j        |j                   t          ||| |           t          |j        |j        | j        |j                   t          ||| |           t          |j        |j        | j        |j                   d S )Nr   rL   )r%   rJ   ry   r   r+   r  r1   r  r!   rJ   ry   r%   r@   rA   )r  r2   )rR   r=   r<   rW   r  r  r@   rA   )rn   r   r   ro   rp   r   r8   s          rF   test_appendr!    s   	...5G5G5GHH	I	IB
,   '9'9'9::BVBVBV  C ,   '9'9'9::BVBVBV  C .Q

C>#q!!D>#q!!D
	1a&qc
333Ac1b!,,,c4S111ceTVRT35999 c4S111ceTVRT35999 c3C000ceSUBD#%888c3C000ceSUBD#%88888rH   c                 z   t          j        g dg dd          t          j        g dg dd          t          j        g dg dd          d} t          dddt          j                    	          }t          j        | d
|d d g          }|                                }t          j        g dg dd          t          j        g dg dd          t          j        g dg dd          d} t          j        | d|d d g          }|                                }t          j        g dg dd          t          j        g dg dd          d} t          dddt          j                    	          }t          j        | d|d d g          }|                                }t          ||||           t          ||||           t          ||||           t          ||||           t          ||||           t          ||||           t          ||||           t          ||||           d S )Nr  r  rL   r  r  r  r  r9   r  r(   r  r  r  r  r  r)   r  r  )rR   r=   r   r<   r   r  )r  r  r   r   rp   r   r   r   s           rF   test_append2r#  	  sx    ,YYYYYY??@@,YYYYYY??@@,YYYYYY??@@ C
 4d++HHHD<S$t55D
,,..C ,\\\EEFF,\\\EEFF,\\\			BBCC C
 <S$t55D
,,..C ,\\\EEFF,\\\EEFF C 4d++HHHD<S$t55D
,,..CdD#s333dD#s333 dD#s333dD#s333 dCc222dCc222 dCc222dCc22222rH   c            
        t          j        t          j        dd          t	          d          t          j        ddd          d          t          j        t          j        dd          t	          d          t          j        ddd          d          g} g }| D ]}|j                            d	          |_        |                                }|j        j        	                    t	          d
                    |_        |j        |_
        |                    |                    |j                             |\  }}dD ]}d | D             }|s)t          |d         j        dgd          |d         _        |\  }}t          ||||          }	t!          |	j
                  |k    sJ t!          |	j                  |k    sJ t          |j        |j        |j        |j                  }	t!          |	j
                  |k    sJ t!          |	          |k    sJ t          |j
        |j
        |j
        |j
                  }	t!          |	          |k    sJ d S )Nr$   r+   r  r  r  )r(   r)   r   r   r  r  r  )TFc                <    g | ]}t          j        |d d          S )r!   Frg   r  )r  r  s     rF   r	  z+test_append_categorical.<locals>.<listcomp>X	  s)    PPPA2>!???PPPrH   r   r)   Tr1   )rR   r=   r  r  r>   r)   r  rZ  r  r  r2   r  r\  r   r  r  r   )
r  frames2rn   r   r   r   r	  r   rp   r  s
             rF   test_append_categoricalr'  ;	  s\    	Yq"%%']]Yq"D111 	
 	
 	Yr2&&']]Yr2T222 	
 	
F  G - -t{{:&&ggii	((e554s}}SU++,,,,HC 2 2PPPPP 	5
 3%t     GAJ 
d'dC==#CI..%7777#CE**e3333'suEE#CI..%7777#C((E1111'
DJ	39UU#C((E11111%2 2rH   c                     t          j        dg dig d          } t          j        | d          }t	          ||| |           }|j        du sJ d S )Nr(   r3   r1   r!   re   F)rR   r=   r<   rW   r  ri   )rn   ro   r  s      rF   test_append_lose_divisionsr)  l	  sd    	sLLL)	>	>	>B
.
+
+
+C
#Cb"
5
5C%''''''rH   c                    t          j        dg dig d          } t          j        | d          }|                    d          }|                    |d          }|j        d	k    sJ |                                 d S )
Nr(   )r    r    r    r  r1   r!   re   r  )r   )r    r    )rR   r=   r<   rW   r\  r   rE   r   )rn   ro   rp   joineds       rF   test_singleton_divisionsr,  u	  s    	sIII&iii	8	8	8B
.
+
+
+C==DYYtSY))Fv%%%%
NNrH   c                     t          j        dg di          } t          j        | d                              d          }|                    dd          }t          ||                     d                     d S )Nr(   r   r   r   r   r!   re   )r   r   T)rE   force)rR   r=   r<   rW   r\  rS   r   rm   s      rF   #test_repartition_repeated_divisionsr0  	  ss    	sLLL)	*	*B
.
+
+
+
5
5c
:
:C??V4?88DdBLL%%&&&&&rH   c                 >   t          j        dg di          } t          j        ddgi          }t          j        | d                              d          }t          j        |d                              d          }|j        dk    sJ t          |          t          |           k    sJ |                    |ddd	          }|                     d          }|                    d          }|                    |ddd	          }t          ||           d S )
Nr(   r.  r   r!   re   r    r{   Tr#  )	rR   r=   r<   rW   r\  rf   r?   r   r   )r   r   r   rp   r1sf1sf2r2s           rF   test_multi_duplicate_divisionsr6  	  s   
,\\\*
+
+C
,aSz
"
"C>#1---77<<D>#1---77<<Dq    t99C    	Df4	H	HB
--

C
--

C	3Ft	F	FBb"rH   c            	     r   d} t          j        g dg dd          }t          j        |d          }t          j        t          j                    d          }t	          d| dz             D ]H}t          t          j        |||j        |k             d	
          ||j        |k             dd           Id S )Nr"   )rX   rY   r   r  Er  )r    r    r!   r!   r"   r"   )userclusterr+   re   r   r    r   r|   F)r   check_divisions)rR   r=   r<   rW   r   r   r   r:  )
k_clustersrn   r_  r(   s       rF   test_merge_outer_emptyr=  	  s    J	///<N<N<NOO
 
B 
	+	+	+B~blnn"===H1j1n%% 
 
HXr"*/2@@@rzQ!		
 	
 	
 	
 	

 
rH   c            	        t          j        dt          j        ddgt          j        t          j                            i          } t          j        dt          j        ddgt          j        t          j                            i          }t          j        d          5 }t          j
                            | |dd           d d d            n# 1 swxY w Y   |rJ d S )Nr@   r    r!   r  Tr  )rR   r=   r  arrayr  r  longlongrx  ry  r<   multiwarn_dtype_mismatch)r   r   r  s      rF   test_dtype_equality_warningrC  	  s    
,RXq!fBHRX4F4FGGGH
I
IC
,RXq!fBHR[4I4IJJJK
L
LC		 	-	-	- 9
$$S#sC8889 9 9 9 9 9 9 9 9 9 9 9 9 9 9s   &#CCCc                   d}d}t          j        t          j                            t          j        |                    t          j                            d|          d          }t          j        t          j                            t          j        |                    t          j                            d|          d          }| dk    rt          j        d          }t          j        d          }|	                    |          }|	                    |          }|
                    ||          }|
                    ||          }n*t          j	        ||          }t          j	        ||          }|                    d	g                                          }	|                    d
g                                          }
t          |	|
gd          }|                    d	g                                          }|                    d
g                                          }t          j                    5  t          j        dt$                     t          j        ||gd          }d d d            n# 1 swxY w Y   t'          |                                                                |                                           d S )Nr%   r"   r7   r  rL   rM   r  r  r@   rN   r    r  rv  )rR   r=   r  re  permutationr  r  rj   r  rW   r  r<   groupbysumr   rx  ry  rz  r  r   r   
sort_index)r  r  rf   d1d2r  r  r  r  
grouped_d1
grouped_d2r  grouped_dd1grouped_dd2res_dds                  rF   test_groupby_concat_cudfrP  	  s    DK	&&ry77""3T"22	
 	

 
B 
&&ry77""3T"22	
 	

 
B  "6**'44	b!!b!!!!"k22!!"k22nR--nR--SE""&&((JSE""&&((J
*j)
2
2
2C++se$$((**K++se$$((**K		 	"	" ? ?h444K5A>>>? ? ? ? ? ? ? ? ? ? ? ? ? ? ? fnn))++S^^-=-=>>>>>s   3JJ
Jorderedc                   t          j        dt          j        g dg d|           i          }t          j        |d          }t          j        dt          j        g dg d|           i          }t          j        |d          }t          j        ||g          }|d                             d          |d<   t          j        ||gd	          }t          ||           d S )
Nr(   )r@   rA   rN   r@   r   )
categoriesrQ  r!   )rN   rA   r@   r  T)ignore_order)rR   r=   Categoricalr<   rW   r   r  r   )rQ  r  r   r  rp   r   r   s          rF   test_concat_ignore_orderrV  	  s    <$$$'  	
 D >$""D<OOOW  	
 D >$""Dy$&&HSM((44HSMYd|$777FfhrH   r  Int64Float64zint64[pyarrow]z:Support for ArrowDtypes requires pyarrow and pandas>=1.5.0zfloat64[pyarrow]c                z   t          j        g dg dt          d          d          }t          j        g dg dt          d          d          }|d                             |           |d<   |d                             |           |d<   t	          j        |d          }t	          j        |d          }|                    |d	          }|                    |d	          }t          ||d
           t          j        ||d	          }t	          j        ||d	          }t          ||d
           d S )Nr  )r    r    r"   aabr   )r@   rL  rM  r@   r!   re   r   Fr   )rR   r=   r>   r  r<   rW   r   r   )r  r   r   r   rp   r  rG  s          rF   test_nullable_types_merger[  	  s2   , ,YYYYYYT%[[II
J
JC
,YYYYYYT%[[II
J
JC3xu%%CH3xu%%CH>#1---D>#1---DYYssY##FZZZ%%Fff%0000Xc33'''FXdDS)))Fff%000000rH   c                    t          j        g dg dd          } t          j        dg dig d          }t          j        | d          }t          j        |d	          }|d
                             d          |d
<   |j                            d          |_        |                                                    |                                d
d          }|                    |d
d          }|j        j	        dk    sJ |                                }|j        j	        dk    sJ t          ||          sJ d S )N)r@   r@   rA   rA   )r   r   r+   r+   )join_colr@   rA   r  r1   r!   re   r    r]  r  r{   r  )rR   r=   r<   rW   r  r2   r   r   r]  r  r   )rn   r   ro   rp   r   actual_daskrG  s          rF   test_categorical_joinr_  %
  sH   	,,,	
 	

 
B ,\\\*2F2F2F
G
G
GC
.
+
+
+C>#1---D*o,,Z88C
O"":..DJ{{}}!!$,,..ZV!LLH((4JF(;;K%3333  ""F? J....Xv&&&&&&&rH   c                 D   t          j        ddgt          j        ddg          d          } t          j        dt          j        ddg          i          }t          j        || dd	          }t	          j        | d
          }t	          j        |d
          }t	          j        ||dd	                                          }|j        j        dk    sJ |j	        j        dk    sJ |j
        j        dk    sJ |j        j        dk    sJ t          ||          sJ d S Nr   r    r@   rA   r  r   TrX   r  r!   re   r  r  rR   r=   rU  r   r<   rW   r   r   r  rY   rX   r2   r   r   r   r   r   rp   rG  s         rF   5test_categorical_merge_with_columns_missing_from_leftrd  >
  s"   
,aV".#s*D*DEE
F
FC
,R^S#J778
9
9CxSTC@@@H>#1---D>#1---DXdDTC@@@HHJJF8>Z''''8>Z''''8>W$$$$<((((Xv&&&&&&&rH   c                 h   t          j        t          j        ddg          t          j        ddg          d          } t          j        dt          j        ddg          i          }t          j        || dd	          }t	          j        | d
          }t	          j        |d
          }t	          j        ||dd	                                          }|j        j        dk    sJ |j	        j        dk    sJ |j
        j        dk    sJ |j        j        dk    sJ t          ||          sJ d S ra  rb  rc  s         rF   Itest_categorical_merge_with_merge_column_cat_in_one_and_not_other_upcastsrf  O
  s,   
,R^QF33".#s:T:TUU
V
VC
,R^S#J778
9
9CxSTC@@@H>#1---D>#1---DXdDTC@@@HHJJF8>Z''''8>Z''''8>W$$$$<((((Xv&&&&&&&rH   c                    t          j        g dg dd          } t          j        g dg dd          }t          j        | d          }|j                            d          |d	<   t          j        |d          }|j                            d          |d	<   |                    |d	
          }|j        j        dk    sJ |                                }|j        j        dk    sJ d S )Nr   r    r!   r"   )r#   r$   r%   rJ   r  )r   r    r!   r#   )r#   r$   rJ   rJ   )rX   r   r!   r  rX   r   )	rR   r=   r<   rW   rX   r  r   r  r   )r@   rA   r   r   r^  rG  s         rF   -test_categorical_merge_retains_category_dtyperi  `
  s    
<<<lll;;<<A
<<<lll;;<<A
.A

Cu||J''CH
.A

Cu||J''CH))CC)((K=*,,,,  ""F8>Z''''''rH   c                 <   t          j        dg dig d          } t          j        dg dig d          }t          j        | d	          }|                    dd
i          }t          |                     |          |                    |                     d S )NrX   r   )r8   vr  )datar2   rY   )trO   r  )rk  r  r  r    re   r  )rR   r=   r<   rW   r  r   r   )r   r   r   s      rF   ?test_categorical_merge_does_not_raise_setting_with_copy_warningrn  r
  s    
,S///2///
J
J
JC
,S///2///
J
J
JC>#1---D
**c:&
'
'CchhsmmTYYs^^,,,,,rH   )r   r{   r   rf   r       baselgsmc           	        d}d}d}d}d}t          j        t          j                            t          j        d          |          t          j        |          d          }t          j        ||          }	t          j        t          j                            t          j        d          |          t          j        |          d          }
t          j        |
|          }|d	k    r	|}|	}|
}|}n|
}|}|}|	}t          j        ||d
| ||d          }t          j        ||d
|           }|d
         	                    t          j
                  |d
<   |d
         	                    t          j
                  |d
<   |r|j        |k    sJ t          |                                                    d
          |                    d
          d           d S )Ni  r  r-   r"   r  r7   r'   re   rq  r)   r  )r   r}   rf   	broadcastr   r  Fr   )rR   r=   r  re  choicer  r<   rW   r   r  int32rf   r   r   r   )r}   rf   rp  size_lgsize_smnpartitions_lgnpartitions_smbroadcast_biasrq  ddf_lgrr  ddf_smr{   ddf_leftr   	ddf_right	dd_result	pd_results                     rF   test_merge_tasks_large_to_smallr  }
  s    GGNNN	!!")C..'::7##	
 	

 
B ^BN;;;F	!!")C..'::7##	
 	

 
B ^BN;;;Ft||			   I u#666I s^**2844IcNs^**2844IcN 4$3333'',,c""     rH   r   r  c                   t          j        ddgdz  t          d          dd          }t          j        ddgdz  t          d	          d
d          }t          j        ||d|           }t	          |j        d          sJ d S )Nr    r!   r6      )r@   b_leftr   re   r+   r,   )r@   b_rightT)rt  r   z
bcast-join)r<   	from_dictr   r   r   r   )r   r{   r   r   s       rF   test_broadcast_truer  
  s     <q!frkU3ZZ@@bQQQDL1v{uRyyAAqQQQEXdET7CCCFV[,///////rH   c                   t          j        t          j        g dg ddg d          d          }t          j        t          j        g dg ddg d          d          t          j        t          j        g d	g d
dg d          d          g}t	          j        t                    5 }|                    ||            d d d            n# 1 swxY w Y   |                    d           d S )Nr  r  rL   r   r    r"   r1   r"   r  )r$   r%   ry   r  r  )r   r   r   r|   z-merge_multi only supports left or outer joins)	r<   rW   rR   r=   rj   rk   rl   r   r   )r}   base_dfr  rL  s       rF   ,test_pairwise_rejects_unsupported_join_typesr  
  sG   n
99999955YYYGGG G 	Lyyyyyy99KKKQ	
 	
 	Lyyyyyy99KKKQ	
 	
	C 
z	"	" #aSc"""# # # # # # # # # # # # # # #GG;<<<<<s   0CCCnpartitions_baser  npartitions_otherc                   g }t          d          D ]R}t          j        | dg d| dg dig d          }t          j        ||          }|                    |           St          j        t          j        g d          |          }|D ]}|                    || 	          }t          j        t          j        g d          |          }|                    || 	          }t          ||           d S )
Nr+   rX   r  rY   r  rh  r1   r  r|   )r   rR   r=   r<   rW   r  r   r   )	r}   r  r  dfs_to_merger  rn   ro   ddf_loopddf_pairwises	            rF   2test_pairwise_merge_results_in_identical_output_dfr  
  s%    L2YY 	! 	!\ ,,
 
 
 nR!233C    ~bl;;;=MNNH / /==#=..>",YYY"?"?"?AQRRL$$\s$;;LlH%%%%%rH   )
__future__r   rx  numpyr  r  rR   rj   packaging.versionr   pandas.api.typesr   dask.dataframe	dataframer<   dask._compatibilityr   	dask.baser   dask.dataframe._compatr   r	   r
   r   r   r   dask.dataframe.corer   dask.dataframe.methodsr   dask.dataframe.multir   r   r   r   r   dask.dataframe.utilsr   r   r   r   r   r   r   dask.utils_testr   r   pyarrowrg  ImportErrorrc   rq   rs   r   r   markparametrizer   r   r   r   r   r   r   r  r  rD  rM  rS  rU  rc  rn  rs  r|  r  r  r  r  xfailr  r  r  r  r  r  paramgpur	  r  slowr;  skipifrD  rH  rW  rr  rj  rq  r{  r}  r  r  r  r  r  r  r  r  r  r  r  r  r  AssertionErrorr
  r  r  r  r!  r#  r'  r)  r,  r0  r6  r=  rC  rP  rV  r[  r_  rd  rf  ri  rn  r  r  r  r  r  rH   rF   <module>r     s   " " " " " "           % % % % % % , , , , , ,       * * * * * * . . . . . .                ' & & & & & ) ) ) ) ) )                               = < < < < < < <   	BBBI  I  I X$ $ $(*- *- *-Z     F0 0 0*  C C CDD  EDD, , ,*  $	' 	' 	'N N NV V V  .u>>&H&H&HII' ' JI ?>'  .u>>&H&H&HII,udm<<< < =< JI ?><~  7' 7' 7't7' 7' 7'tE' E' E'P	 	 	17 17 17h 68RS 7 7 7.	$ 	$ 	$ '7!344
 
 54
. '7!344$ $ 54$6 '7!344$ $ 54$6     .B B B2 =)EeTT	nbn\**+	
NBN<(()	|,,-
 
; ;
 
 UT;,  C C CDDdE]33 1  1 43 ED 1F  C C CDDJ J EDJb ' 233D D 43D&+ + +( ":":":;;Z 899+##E $  	
 	
 	
 	V6;?333 #1 #1  :9 <;#1L	5 	5 	5  C C CDDj j ED jZ M*EFF C C CDDt t ED  GFtn       C C CDDK K ED K\ 
U999TuenIII&&&t%%% 4 4 4*O O O %//; ; 0/;0  (  122%//    0/ 32 .     <	H 	H 	H	  	  	   "6 6 6' ' '  (2 2 25K 5K 5Kp% % %P,! ,! ,!^Z
 Z
 Z
z !+##@wwx/@/@!@W%	 $  	
	
 
	
 
	
 	+##@wwx/@/@!@W%	 $  	
	
 
	
 
	
 	5 >I4 I4? >I4X          2   M*ABB9 9 CB9> M*ABB'3 '3 CB'3T M*ABB-2 -2 CB-2` M*ABB( ( CB(  ' ' '  $
 
 
$   xfFKODDDE +? +? +?\ T5M22    32 . +$$d
/-/S %  	
 	
 	
 	+$$d
/-/S %  	
 	
 	
 *1 1+ *1$' ' '2' ' '"' ' '"( ( ($- - -  : : :;;R11$..8 8 /. 21 <;8v T7O440 0 540 ' 233= = 43=$  122+YYY77,iii88& & 98 87 32& & &s   :A? ?B	B	