
    >ie                    L1   d dl mZ d dlZd dlZd dlZd dlZd dlZd dlmZ d dl	m
Z
 d dlZd dlZd dlZd dlmZ d dlZd dlmZ d dlZd dlmZ d dlmZmZ d dlmZmZm Z  d d	l!m"Z" d d
l#m$Z$ d dl%m&Z& d dl'm(Z(m)Z) d dl*m+Z+ d dl,m-Z- d dl.m/Z/ 	 d dl0Z0 ee0j1                  Z2n# e3$ r dZ0 ed          Z2Y nw xY w	 d dl4Z5 ee5j1                  Z6n# e3$ r dZ5 ed          Z6Y nw xY w	 d dl7m8Z9 n# e3$ r dZ9Y nw xY we0 Z:ej;        <                    e:d          Z=e9 Z>dZ?ej;        <                    e>e?          Z@dZAdZB ejC        d  eDeA          D             d  eDeA          D             d ejE        d  eDeA          D             d                    ZF ejG        eFeB          ZH ejI         ejJ        de=            ejJ        d!e@           g"          d#             ZKd$ ZL eL            ZMe@d%             ZNe=d&             ZO eL            ej;        P                    d'dd(g          d)                         ZQej;        P                    d*dd(g           eL            d+                         ZR eL            d,             ZS eL            d-             ZT eL            d.             ZU eL            d/             ZV eL            d0             ZW eL            d1             ZX eL            d2             ZYd3 ZZ eL            d4             Z[ eL            d5             Z\d6 Z] eL            d7             Z^d8 Z_d9 Z` eL            d:             Za eL            d;             Zbe@ej;        P                    d<d= ejJ        d!ej;        <                    e d>                     g          d?                         Zce@d@             Zd eL            dA             Zeej;        P                    dBdd(g          dC             ZfdD ZgdE ZhdF Zie@ej;        P                    dGdd(g          ej;        P                    dHdd(g          dI                                     ZjdJ ZkdK Zlej;        P                    dBdd(g          dL             Zmej;        P                    dBdd(g          ej;        P                    dM ejn        dNdOdPQ           ejo        dRS          f ejp        d dTdR          dUfg          dV                         ZqdW Zrej;        P                    dBdd(g          dX             Zsej;        jt        dY             Zu eL            dZ             Zvd[ Zwej;        P                    d\ ejC        d]g d^i          i i f ejC        d]g d_i          i i f ejC        d]g d`i          i i f ejC        d]g dai          dbe5rd] e5jx                    inddci f ejC        d] ejy        g dd          i          i ded]gif ejC        d] ejy        g df          i          i ded]gif ejC        d] ez e{ej|        g dg                    i          i i f ejC        d]g dgi          }                    dh          i i f ejC        d]g d^i          }                    dh          i i f ejC        d]g d^i          }                    di          i i f ejC        d]g d^i          }                    dj          i i f ejC        d]g dgi          }                    dk          i i f ejC        d]g dgi          }                    dl          i i f ejC        d]g dgi          }                    dm          i i f ejC        d]g d^i          }                    dn          i i f ejC        d]g d^i          }                    do          i i f ejC        d]g dpig d^          i i f ejC        d]g dqi ejE        g drds                    i i f ejC        g drg d^d          i i f ejC        g drg d^ddtd]gu          i i f ejC        dg d^i          i i f ejC        d]g dvi          i i f ejC        dwg dxi          i i f ejC        dyg dxi          i i f ejC        dzg dxi          i i fg          ej;        jt        d{                         Z~ej;                             e)            oe6 ed|          k     d}          d~             Zej;        j        d             Zd Zej;        P                    dd(dg          d             Ze@e=d                         Ze@ej;        jt        d                         Ze@ej;        P                    d*dd(g          ej;        P                    dddg          d                                     Ze@d             Ze@d             Zd Zd Ze@ej;        P                    dddgg          d                         Z eL            d             Z eL            d             Z eL            d             Zd Zd Zd Ze=d             Ze=d             Ze=ej;                            d          d                         Zej;        P                    dddg          d             Ze@ej;        P                    ddd(g          d                         Ze=d             Ze=d             Zd Zd Zd Zej;        P                    dg d          d             Zej;        P                    dg d          d             Z ejI        ddddddddddgdgddddddddddddgdgdddddiddddgddddddddddddgdgddg"          d             Zd Zd Ze@d             Zd Zd Zd Zd Zd Zd Zd Ze@d             Zd ZdÄ Ze@ej;        P                    d ejC        d]g dŢi           ejC        d]g d_i           ejC        d]g d`i           ejJ         ejC        d] ejy        g dd          i                     ejJ         ejC        d] ejy        g df          i                     ejC        d] ez e{ej|        g dƢ                    i           ejC        d] ez e{ej|        g dg                    i           ejC        d]g dgi          }                    dh           ejJ         ejC        d]g d^i          }                    di          ej;                            eoe6 edǦ          k     dȬ                      ejJ         ejC        d]g d^i          }                    dj          ej;                            eoe6 edǦ          k     dȬ                      ejC        d]g d^i          }                    dn           ejC        d]g d^i          }                    do           ejC        d]g dpig d^           ejC        d]g dŢi ejE        g dɢds                     ejC        g drg d^d           ejC        g drg d^ddtd]gu           ejC        dg d^i           ejC        d]g dvi           ejC        dwg dxi           ejC        dyg dxi           ejC        dzg dxi          g          dʄ                         Zd˄ Zd̄ Ze=d̈́             Z eL            d΄             Z eL            dτ             Zej;        P                    dd(dg          ej;        P                    dd(dg           eL            d҄                                     Z eL            dӄ             Zej;        P                    dddg          dԄ             ZdՄ Zej;        P                    dd(dg          ej;        P                    d*dej                            dצ          g          d؄                         Zdل Zdڄ Zdۄ Zd܄ Zd݄ Zdބ Zd߄ Ze@d             Ze@ej;        P                    ddRdg          ej;        P                    dd(dg          d                                     Ze@ej;        P                    dddg          d                         Ze@ej;        P                    d ejJ        dgd dej;        j                   dgd dfg          ej;        P                    dd(dg          d                                     Ze@ej;        P                    dd(dg          d                         Ze@d             Zd Zd Z eL            d             Ze@ej;        P                    dd(dg          ej;        P                    dddg          ej;        P                    dddg           eL            d                                                             Z eL            ej;        P                    dddg          d                         Ze@ej;        P                    dd(dg          ej;        P                    dg d          d                                     Ze@ej;        P                    dd(dg          ej;        P                    dg d           d                                     Z eL            d             Ze=d             Zϐd Zej;        P                    ddd(g           eL            d                         Ze@d             Ze@d             Ze@d             Zej;        P                    d	d
dgg dg          d             Ze@ej;        P                    d	dgd
dgg          d                         Ze@d             ZeMd             Zؐd Zِd Z eL            d             Zېd Zܐd Ze@d             Ze@ej;        P                    dd(dg          d                         Ze@ej;        P                    dddg          d                         Ze@d             Ze@ej;        P                    dd(dg          d                         Zd Ze@d             Ze@d             Ze@ej;        P                    dddg           eL            d                                     Zd  Zd! Zd" Zd# Zd$ Zd% Zd& Ze@d'             Zd( Zd) Zej;        P                    dg d*          d+             Zej;        P                    d,d(dg          ej;        P                    d-dd g          d.                         Ze@ej;        P                    dd/          d0                         Zd1 Zd2 Zd3 Zd4 Zd5 Zd6 Ze@d7             Zej;        P                    d8dRhdRgd9fd:;          d<             Zd= Zej;        P                    d8d>gd>ggd?d>gd?d>ggfd@;          dA             ZdB Zej;        j        dC             Z e@dD             Zej;        P                    dEdFdg          dG             Ze@ej;        P                    dEdHdg          dI                         Ze@ej;        j        dJ                         Ze@dK             ZdL ZdM Zej;        P                    dNd(dg          ej;        <                    e dO          dP                         Z	e@ej;        <                    e dO          dQ                         Z
e@ej;        P                    d<dRd!g          ej;        <                    e d>          dS                                     Ze@ej;        <                    e dT          dU                         Ze@ej;        <                    e dV          dW                         Ze@ej;        P                    dXddYggg          dZ                         ZdS ([      )annotationsN)Decimal)	MagicMock)parse)
_numpy_124)	Blockwiseoptimize_blockwise)PANDAS_GE_150PANDAS_GE_200PANDAS_GE_202)
get_engine_parse_pandas_metadata)optimize_dataframe_getitem)	assert_eqpyarrow_strings_enabled)DataFrameIOLayer)natural_sort_key)	hlg_layerF0z4fastparquet not found or pyarrow strings are enabledreasonzpyarrow not found(      c                    g | ]
}|d z  dz  S )       .0is     Dlib/python3.11/site-packages/dask/dataframe/io/tests/test_parquet.py
<listcomp>r#   B   s     ...Aa!eai...    c                    g | ]}|d z  S )g      @r   r   s     r"   r#   r#   C   s    ,,,!a#g,,,r$   xyc                    g | ]}d |z  S )
   r   r   s     r"   r#   r#   E   s    111qBF111r$   myindexnameindexnpartitionsfastparquetmarkspyarrow)paramsc                    | j         S Nparamrequests    r"   enginer=   K   s     =r$   c                   	 ddht           t          d		fdD             }|                                 D ]\  }}|                    dd          \  }}t	          |                    d                    }|dvs%t          |          dk    st          |          z
  rt          d	|z             t          t          j
        |          |
          }t          |          dk    r||                             |           |D ]!}||v r||                             |           "t          j
                            dd t          |                                          D                       S )zProduct of both engines for write/read:

    To add custom marks, pass keyword of the form: `mark_writer_reader=reason`,
    or `mark_engine=reason` to apply to all parameters with that engine.r5   r2   )r2   r5   c                @    i | ]}D ]}||f|         |         gS r   r   )r    wrbackends
skip_markss      r"   
<dictcomp>z&write_read_engines.<locals>.<dictcomp>a   s9    XXXxXX!aVjmZ]3XXXXr$   _   )xfailskip   zunknown keyword %rr   )write_engineread_enginec           	     P    g | ]#\  }}t          j        |d t          |          i$S r3   )pytestr:   tuple)r    kvs      r"   r#   z&write_read_engines.<locals>.<listcomp>s   s1    NNNfq!q	)a	)	)NNNr$   )FASTPARQUET_MARKPYARROW_MARKitemssplitrN   lenset
ValueErrorgetattrrM   markappendparametrizesorted)
kwargsr4   kwvalkindrestkeyrO   rB   rC   s
           @@r"   write_read_enginesrc   U   s   
 =)H ( J YXXXXXXXXE <<>> ) )CXXc1%%
dDJJsOO$$(((CHHqLLCHHx<OL1B6777(gfk4((444s88q==#Jc"""" ) )!88!HOOC((() ;""'NNu{{}}8M8MNNN  r$   c                 f    ddl m}  t          d          | k    sJ t          d          | k    sJ d S )Nr   ArrowDatasetEnginer5   arrow)dask.dataframe.io.parquet.arrowrf   r   re   s    r"   test_get_engine_pyarrowri   z   sN    BBBBBBi  $66666g"4444444r$   c                 <    ddl m}  t          d          | k    sJ d S )Nr   FastParquetEnginer2   )%dask.dataframe.io.parquet.fastparquetrl   r   rk   s    r"   test_get_engine_fastparquetrn      s4    GGGGGGm$$(9999999r$   has_metadataTc           	     >   t          |           }t          j        t          j        dt          j                  t          j        dt          j                  t          j        dt          j                  t          j        	                    g dd          
                    d          d          }t          j        |d          }|rd	d
ini } |j        |fd|d| t          j        |          }d|v |k    sJ d|v |k    sJ d|v sJ t          j        |d|          }	t#          |	j                  dk    sJ |	                    d                                          }
|j        D ](}||         |
|         k                                    sJ )d S )N  dtypehelloyopeoplesizeOi32i64fbhello  	chunksizewrite_metadata_fileTFwrite_indexr=   _common_metadata	_metadatapart.0.parquetr/   r=   rF   sync	scheduler)strpd	DataFramenparangeint32int64float64randomchoiceastypeddfrom_pandas
to_parquetoslistdirread_parquetrU   	divisionscomputereset_indexcolumnsall)tmpdirrJ   rK   ro   tmpdatadfr]   filesdf2outcolumns               r"   
test_localr      s    f++C<9T2229T2224rz222i&&'@'@'@t&LLSS 		
 	
	 	D 
	,	,	,B.:B#T**FBM#H5HHHHHJsOOE%'L88885 \1111u$$$$
/#U;
?
?
?Cs}!!!!
+++
'
'
3
3
5
5C* 3 3VF+002222223 3r$   r/   c                F   t          |           }t          j        g dg dd          d d         }|r|                    dd          }t	          j        |d	          }|                    |||d
           t	          j        ||          }t          ||           d S )Nabr      r      r   r   r   r   TdroprI   r0   r   r=   r   r=   	r   r   r   	set_indexr   r   r   r   r   )r   rJ   rK   r/   fnr   ddfread_dfs           r"   
test_emptyr      s     
VB	OOO)))<<	=	=bqb	AB *\\#D\))
.
+
+
+CNN25SWNXXXob555Gc7r$   c                4   t          |           }t          j        g dg dd          }|                    dd          }t	          j        |d          }|                    ||	           t	          j        |dg|d
          }t          ||           d S )Nr   r   r   r   Tr   rI   r0   r   r/   r=   calculate_divisionsr   )r   rJ   rK   r   r   r   r   s          r"   test_simpler      s    	VB	OOO)))<<	=	=B	c	%	%B
.
+
+
+CNN2lN+++o
3%  G c7r$   c                   t          |           }t          j        g dg dd          }|                    dd          }t	          j        |d          }|                    ||d	d	
                                           t          j	        |          }d|vsJ t	          j
        t          j                            |d          dg|d          }t          ||           d S )Nr   r   r   r   Tr   rI   r0   F)r=   r   r   r   	*.parquetr   )r   r   r   r   r   r   r   r   r   r   r   pathjoinr   )r   rJ   rK   r   r   r   r   r   s           r"   test_delayed_no_metadatar      s    	VB	OOO)))<<	=	=B	c	%	%B
.
+
+
+CNN
<E   giiiJrNNEe####o
R%%e 	  G c7r$   c                   t          |           }t                              ||           t          j                            t          j                            |d                    r2t          j        t          j                            |d                     t          j        |          }d|vsJ t          j
        t          j                            |d          |dd          }t          t          |           d S )Nr   r   r   r+   Tr=   r/   r   r   r   r   r   r   existsr   unlinkr   r   r   r   )r   rJ   rK   tmp_pathr   ddf2s         r"   test_read_globr      s    6{{HNN8LN111	w~~bgll8[99:: 7
	"',,x55666Jx  Ee####?
X{++ 	  D c4r$   c                    t          |           }t                              |d|           t          j        ||dd          }t          t          |dd           d S )NFr   r   check_indexcheck_divisionsr   r   r   r   r   r   r   rJ   rK   r   r   s        r"   test_calculate_divisions_falser      sf    6{{HNN8|NDDD?!	  D c4UEBBBBBBr$   c                    ||cxk    rdk    r'n n$t           j        dk    rt          j        d           t	                      t
                               |           t           fdt          j                   D             t                    }t          j        ||dd	          }t          t
          |           d S )
Nr2   ntzfilepath bug.r   c              3     K   | ]9}|                     d           t          j                            |          V  :dS )r   N)endswithr   r   r   )r    r~   r   s     r"   	<genexpr>z!test_read_list.<locals>.<genexpr>  sY       	
 	
::k**	
GLL##	
 	
 	
 	
 	
 	
r$   )rb   r+   Tr   )r   r-   rM   rH   r   r   r   r\   r   r   r   r   r   )r   rJ   rK   r   r   s   `    r"   test_read_listr      s    {3333m333334 	O$$$[[FNN6,N///	
 	
 	
 	
Z''	
 	
 	

   E ?k  D c4r$   c                <   t          |           }t                              ||           t          t	          j        |g |d          t          g                     t          t	          j        |g |d          t          g                                          d           t          t	          j        |dg|d          t          dg                    t          t	          j        |dg|d          t          dg                                         d           d S )Nr   Tr   r=   r   Fr   r'   r   r   r   r   r   r   clear_divisionsr   rJ   rK   r   s       r"   test_columns_auto_indexr     s5   	VBNN2lN+++ 
B{PTUUUB   
B{PUVVVB!!    
kt	
 	
 	
 	SE
	   
ku	
 	
 	
 	SE
""$$     r$   c           	     $   t          |           }t                              ||           t          t	          j        |g |dd          t          g                     t          t	          j        |g |dd          t          g                                          d           t          t	          j        |ddg|d          t          dg                    t          t	          j        |ddg|d          t          dg                                         d           t          t	          j        |ddd	g|d          t                     t          t	          j        |ddd	g|d          t                                          d           d S )
Nr   r+   T)r   r=   r/   r   Fr   r'   r/   r   r=   r   r(   r   r   s       r"   test_columns_indexr   9  s   	VBNN2lN+++ 
 $	
 	
 	
 	B	 	 	 
 %	
 	
 	
 	B!!
 
 
 
 
E $	
 	
 	
 	SE
	 	 	 
E %	
 	
 	
 	SE
""$$
 
 
 
 
#J $	
 	
 	
 		 	 	 
#J %	
 	
 	
 	
 
 
 
 
 
r$   c                   t          |           }t                              ||           t          j        t
          t          f          5  t          j        |dg|           d d d            n# 1 swxY w Y   t          j        t          t          f          5  t          j        |dgt          t          j                  z   |           d d d            d S # 1 swxY w Y   d S )Nr   	nonesenser   r=   )r   r   r   rM   raisesrW   KeyErrorr   r   	Exceptionlistr   r   r=   r   s      r"   test_nonsense_columnr     sl   	VBNN2fN%%%	
H-	.	. B B
[M&AAAAB B B B B B B B B B B B B B B		8,	-	- V V
[MD4E4E$EfUUUUV V V V V V V V V V V V V V V V V Vs$   A11A58A53CC #C c           	        t          |           }t                              ||           t                                          }t	          t          j        |d|d          |dd           t	          t          j        |dddg|d          |ddg         dd           t	          t          j        |dd	dg|d          |d	dg         dd           d S )
Nr   FTr   r   r'   r(   r   r+   )r   r   r   r   r   r   r   )r   rJ   rK   r   r   s        r"   test_columns_no_indexr     s   	VBNN2lN+++??D
 
%QUVVV	    
#J $	
 	
 	
 	c3Z    
$ $	
 	
 	
 	i     r$   c                    t          |           }t                              ||d           t          j        ||d          }|j        j        J |j        rJ d S )NFr=   r   r=   r/   )r   r   r   r   r   r/   r-   known_divisions)r   rJ   rK   r   r   s        r"   !test_calculate_divisions_no_indexr     s`    	VBNN2lN>>>	Ku	=	=	=B8=   !!!!!!r$   c           	     x   t           j                            t          |           d          }t          j                            t          j        d          t          j        d          dz   gddg          }t	          j	        t          j
                            dd          dd	g|
          }|                    d          }|dk    r*t          j        ||                                d           nFt          j        t"          j                            |                                d          |           t)          j        |||j                  }t/          ||           t)          j        |d||j                  }t/          ||d                    t)          j        |dd	gddg|          }t/          ||                    dd	g          ddg                    t)          j        |d|          }t/          ||           t)          j        |d	gdg|          }t/          ||                    d          d	g                    t)          j        |dd	gdg|          }t/          ||                    d          dd	g                    t)          j        |ddgdg|          }t/          ||                    d          ddg                    t)          j        |ddd	g|          }t/          ||dd	g                    dD ]G}t)          j        ||ddg|          }t/          ||                    |          ddg                    HdD ]M}t3          j        t6                    5  t)          j        ||ddg|          }d d d            n# 1 swxY w Y   Ndd|                    d          fdd	|fdd|dg         fdd|                    d          dg         fdd	|                    d          ffD ]4\  }}	}
t)          j        |||	|          }t/          ||
|	                    5d S )Ntest.parquetr*   rF   x0x1namesrI   r   r   )r   r/   Fr   r2   )r   preserve_indexr   )r   r=   r/   )r/   r   r=   r   )r   r/   r=   )r   r   )r   r   )r   r   r   r   r   
MultiIndexfrom_arraysr   r   r   r   randnr   r2   writepqwrite_tablepaTabler   r   r   r   r   r   rM   r   rW   )r   r=   r   r/   r   r   r   dindcolsol_dfs              r"   #test_columns_index_with_multi_indexr
    su   	c&kk>	2	2BM%%	2	")*4, &  E 
bioob!,,sCj	N	N	NB
..e.
$
$C"bnn..EBBBBB 	rx++BNN,<,<U+SSUWXXX
/"V5;
?
?
?Cc2
CekJJJAaC
3*tTl6RRRAaSz**D$<8999 	%777Aa
SE#vFFFAas##SE*+++
S#JtfVLLLAat$$c3Z0111 	T3KvfMMMAat$$dC[1222 	%$VLLLAadC[!""" 8 8OBedC[PPP!S]]5))4+67777  U U]:&& 	U 	U%$VTTTA	U 	U 	U 	U 	U 	U 	U 	U 	U 	U 	U 	U 	U 	U 	U
 
tS]]4(()	S	c4&k"	dCMM#&&v./	c3==%%& " "S& OBc3vFFF!VC[!!!!" "s   5NN 	#N 	c                   t          |           }t          j        g dg dd          }t          j        |d          }|                    ||           t          j        ||          }t          ||d           d S )	NrF   rI      r   r   rI   r0   r   Fr   )r   r   r   r   r   r   r   r   )r   rJ   rK   r   r   r   r   s          r"   test_no_indexr    s    	VB	IIIIII66	7	7B
.
+
+
+CNN2lN+++?2k222Db$E******r$   c                .   t          |           }t                              ||           t          j        |dgd|d          }t          t          dg         |           t          j        |dd|d          }t          t          j        |           d S )Nr   r'   r+   T)r   r/   r=   r   )r   r   r   r   r   r   r'   )r   r=   r   r   s       r"   test_read_seriesr    s    	VBNN2fN%%%?
SE6t  D c3%j$?
CyT  D ceTr$   c                   t          |           }t                              |           fd}t           ||          j                  t           ||          j                  k    sJ t           ||          j                  t           ||dg          j                  k    sJ t           ||d          j                  t           ||dg          j                  k    sJ d S )Nr   c                ,    t          j        | fdi|S )Nr=   r   r   )r   r]   r=   s     r"   readztest_names.<locals>.read)  s     r;;&;F;;;r$   r'   r   r'   )r   r   r   rV   dask)r   r=   r   r  s    `  r"   
test_namesr  %  s    	VBNN2fN%%%< < < < < ttBxx}TT"XX]!3!33333ttBxx}TT"se%<%<%<%A!B!BBBBBttB''',--TT"se5L5L5L5Q1R1RRRRRRRr$   c                >   t          |                     d                    }t                                          }d|j        _        |                    ||                    d          rdnd           t          j	        |d|          }t          ||           d S )Nr   r/   r5   r2   r   r   r   r   r   copyr/   r-   r   
startswithr   r   r   )r   rJ   rK   r   dfpr   s         r"   test_roundtrip_from_pandasr  3  s    	V[[((	)	)B
''))CCINNN
 7 7	 B BU99     /"GK
@
@
@Cc3r$   c                6   |dk    s|dk    rt          j        d           t          j        t          j        ddt          j        ddgd          t          j        d	t          j        d
d	d
gd          t          j        dddt          j        dgd          t          j        ddddt          j        gd          d          }t          j        |d          }|                    | |           t          j	        | |          }t          ||           dS )zx
    Test round-tripping nullable extension dtypes. Parquet engines will
    typically add dtype metadata for this.
    r2   z.https://github.com/dask/fastparquet/issues/465rF   rI   r  r   Int64rr   TFboolean皙?皙?333333?皙?Float64r   r   cr  stringr   r   r(  r  r0   r   N)rM   rG   r   r   SeriesNAr   r   r   r   r   )r   rJ   rK   r   r   r   s         r"   test_roundtrip_nullable_dtypesr-  ?  s    m##|}'D'DEFFF	Aq"%A.g>>>D"%e<INNNCc25#6iHHHCc36hGGG		
 	

 
B .
+
+
+CNN8LN111?8K888Db$r$   dtype_backendpandasz'Requires pyarrow-backed nullable dtypesc                    |dk    rdnd}t          j        t          j        ddt           j        ddgd| 	          t          j        d
t           j        dd
dgd| 	          t          j        dddt           j        dgd| 	          t          j        ddddt           j        gd| 	          d          }t	          j        |d          }t          j         fd            |                                }t          j	        fdt          |          D                        |dk    rMt          j        t          d          5  t	          j         |d
           ddd           dS # 1 swxY w Y   dS t          j                            d |i          5  t          j        t"                    5  t	          j         |!          }t%          ||           ddd           n# 1 swxY w Y   t          j        t(          d"          5  t	          j         |d
          }t%          ||d#           ddd           n# 1 swxY w Y   ddd           dS # 1 swxY w Y   dS )$w
    Test reading a parquet file without pandas metadata,
    but forcing use of nullable dtypes where appropriate
    r/   	[pyarrow]rF   rI   r  r   r!  rr   TFr"  r#  r$  r%  r&  r'  r   r   r(  r  r)  r*  r0   c                    t           j                            |                               i           }t	          j        |d| dz             dS z0Write a parquet file without the pandas metadatapart..parquetNr  r  r   replace_schema_metadatar  r  r   r!   tabler   s      r"   write_partitionz1test_use_nullable_dtypes.<locals>.write_partitionu  Q     $$R((@@DD
uh)<)<)<)<<=====r$   c                .    g | ]\  }} ||          S r   r   r    r!   pr<  s      r"   r#   z,test_use_nullable_dtypes.<locals>.<listcomp>~  )    JJJDAq//!Q''JJJr$   r2   z&`use_nullable_dtypes` is not supportedmatch)r=   use_nullable_dtypesNzdataframe.dtype_backendr   rD  r  )r   r   r+  r,  r   r   r  delayed
to_delayedr   	enumeraterM   r   rW   r   configrV   AssertionErrorr   warnsFutureWarning	r   r.  r=   dtype_extrar   r   
partitionsr   r<  s	   `       @r"   test_use_nullable_dtypesrO  V  s   $ &11""{K	Aq"%A.6Kk6K6KLLLrueT519P;9P9P   Cc25#6>U>U>UVVVCc36>T{>T>TUUU	
 	
	
 	
B .
+
+
+C	\> > > > \> !!JLJJJJIj4I4IJJJKKK ]:-UVVV 	O 	OOHVNNNN	O 	O 	O 	O 	O 	O 	O 	O 	O 	O 	O 	O 	O 	O 	O 	O 	O 	O [__7GHH 
	7 
	7~.. $ $x???"d###$ $ $ $ $ $ $ $ $ $ $ $ $ $ $
 m3HIII 7 7V   "d6666	7 7 7 7 7 7 7 7 7 7 7 7 7 7 7
	7 
	7 
	7 
	7 
	7 
	7 
	7 
	7 
	7 
	7 
	7 
	7 
	7 
	7 
	7 
	7 
	7 
	7sl   E22E69E6 I!:'G-!I!-G1	1I!4G1	5I!*I	=I!	I	I!I	I!!I%(I%c                $   t          j        t          j        ddt           j        ddgd          t          j        dt           j        dddgd	          t          j        d
ddt           j        dgd          t          j        ddddt           j        gd          d          }t	          j        |d          }|                    | |           t          j                    t          j	                    i}t	          j
        | ddd|j        i          }|                    dt          j	                    i          }t          j        dk    r0|j                            t          j	                              |_        t!          ||           d S )NrF   rI   r  r   r!  rr   TFr"  r#  r$  r%  r&  r'  r   r   r(  r  r)  r*  r0   r   r5   numpy_nullabletypes_mapperr=   r.  arrow_to_pandas   )r   r   r+  r,  r   r   r   r  r   Float32Dtyper   getr   pyarrow_versionmajorr/   r   )r   r=   r   r   rR  resultexpecteds          r"   *test_use_nullable_dtypes_with_types_mapperr\    st    
Aq"%A.g>>>D"%e<INNNCc25#6iHHHCc36hGGG		
 	

 
B .
+
+
+CNN8FN+++ 	

BO%%L _&')9:	  F yy#r00122H"" "..r/@/@AAfhr$   c                >   |dk    r|dk    rt          j        d           t          |           }t          j        dg ddz  id          }t          j        |d	
          }t          j        |||           t          j        |d|          }|	                                j
        j        j                                        g dk    sJ t          j        |dg|          }|	                                j
        j        j                                        g dk    sJ |dk    rt          j        ||          }|	                                j
        j        j                                        g dk    sJ |j        d d         	                                 t          ||          sJ t          j        |g |          }|j        d d         	                                 |j
        |j
        	                                k                                    sJ d S )Nr2   r5   zKnown limitationr'   r   r   r(  d   categoryrr   r  r0   r   
categoriesr=   rq   )rM   rG   r   r   r   r   r   r   r   r   r'   catrb  tolistlocr   r   r   rJ   rK   r   r   r   r   s          r"   test_categoricalrg    s   }$$	)A)A'(((
f++C	sOOOc12*	E	E	EB
.
+
+
+CM#s<0000?33{CCCD<<>>*1133FFFF?3C5EEED<<>>*1133FFFF m##s;777||~~#.5577???JJJJ$!!!T""""" ?32kBBBDHUdUODDFNN$$$))+++++++r$   metadata_filec           	     n   t          |           }t          j        t          j        dt          j                  t          j        dt          j                  t          j        dt          j                  t          j        	                    g dd          
                    d          d          }d|j        _        t          |          dz  }t          j        |j        d	|         d
          }t          j        |j        |d	         d
          }|                    |||           |r\t%          t          |                     d                    d          5 }|                                }	d	d	d	           n# 1 swxY w Y   |                    |d|           |rdt%          t          |                     d                    d          5 }|                                }
d	d	d	           n# 1 swxY w Y   |
|	k    sJ t          j        ||          }t-          ||           d	S )5Test that appended parquet equal to the original one.rq   rr   rt   rx   rz   r{   r/   rI   Nr_  r   r=   r   r   rbTrZ   r=   r   )r   r   r   r   r   r   r   r   r   r   r   r/   r-   rU   r   r   ilocr   openr   r  r   r   )r   r=   rh  r   r   halfddf1r   r~   	metadata1	metadata2ddf3s               r"   test_appendru    so    f++C	9T2229T2224rz222i&&'@'@'@t&LLSS 		
 	
	
 	
B BHMr77a<D>"'%4%.C888D>"'$%%.C888DOOCMOJJJ !#fkk+..//66 	!!I	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	!OOCVO444 &#fkk+..//66 	!!I	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	!I%%%%?3v...Db$s$   %FF
F
G<<H H c           	        t          |           }t          j        t          j        dt          j                  t          j        dt          j                  t          j        dt          j                  t          j        	                    g dd          
                    d          d          }d|j        _        t          |          dz  }t          j        |j        d	|         d
          }t          j        |j        |d	         d
          }|                    |d|           |                    |d|           t          j        ||          }t'          ||           d	S )rj  rq   rr   rt   rx   rz   r{   r/   rI   Nr_  r   Trm  r   )r   r   r   r   r   r   r   r   r   r   r   r/   r-   rU   r   r   rn  r   r   r   )r   r=   r   r   rp  rq  r   rt  s           r"   test_append_createrw    sF   6{{H	9T2229T2224rz222i&&'@'@'@t&LLSS 		
 	
	
 	
B BHMr77a<D>"'%4%.C888D>"'$%%.C888DOOHT&O999OOHT&O999?8F333Db$r$   c           	     @   t          |           }t          j        t          j        ddd          t          j        ddd          t          j        ddd          d          }d	|j        _        t          j        t          j        ddd          t          j        ddd          t          j        d
dd          d          }d	|j        _        |d                             d          |d<   t          j        |j	        |j        d         df<   |d                             d          |d<   t          j        |d          }t          j        |d          }t          j        ||dg|           t          j        ||dgdd|           t          j        ||d	d                                          }|j                            d          |d<   t!          |                    d          t          j        ||g          |j                 d           d S )Nr   r*   r   rr      r_  n   )latlonvaluer/   x      r{  r!  rF   r0   r|  partition_onr=   T)r  rZ   ignore_divisionsr=   r   r}  Fr  )r   r   r   r   r   r/   r-   r   nanre  r   r   r   r   r   r|  r   sort_valuesconcatr   )r   r=   r   df0df1dd_df0dd_df1r   s           r"   test_append_with_partitionr    s"   
f++C
,9Q'2229R7333YsCw777	
 	
 C CIN
,9R73339R7333YsCw777	
 	
 C CIN U""7++CJ#%6CGCIaL% U""7++CJ^CQ///F^CQ///FM&#UGFCCCCMW    /F't  gii  ((CJ  ")S#J"7"7"DRW     r$   c                   t          |           }t          j        t          j                            d          t          j                            g dd          t          j                            g dd          d          }t          j        |d          }|	                    |dg|           t          j
        ||          }t          |j        j        j                  h d	k    sJ d S )
N2   r'   r(   zrx   r^  rI   r   r  r   >   r'   r(   r  r   r   r   r   r   randr   r   r   r   r   rV   r   rc  rb  r   r=   r   r  r   s        r"   test_partition_on_catsr  =  s    
f++C
##!!///!;;!!///!;;	
 	
	 	A 	q!ALLC5L888	V	,	,	,Brtx"##666666r$   metastatsc                   t          |           }t          j        t          j                            d          t          j                            g dd          t          j                            g dd          d          }t          j        |d          }|	                    |dgd|           t          j
        |d|	          }t          |j        j        j                  h d
k    sJ d S )Nr  r  rx   r^  rI   r   r5   r  r=   r   r=   r   >   r'   r(   r  r  )r   r  r  r   r  r   s         r"   test_partition_on_cats_pyarrowr  L  s     f++C
##!!///!;;!!///!;;	
 	
	 	A 	q!ALLC5PTLUUU	YE	J	J	JBrtx"##666666r$   c                   t          |           }t          j        t          j                            d          t          j                            g dd          t          j                            g dd          d          }t          j        |d          }|	                    |dg|d           t          j
        ||d	d
          }t          |j        j        j                  h dk    sJ d S )Nr  r  rx   r^  rI   r   Fr  TrF   r=   r   metadata_task_size>   r'   r(   r  r  r  s        r"    test_partition_parallel_metadatar  ^  s     f++C
##!!///!;;!!///!;;	
 	
	 	A 	q!ALLC5ULSSS	F
 
 
B rtx"##666666r$   c                   t          |           }t          j        t          j                            d          t          j                            g dd          t          j                            g dd          d          }t          j        |d          }|	                    |ddg|           t          j
        ||	          }t          |j        j        j                  h d
k    sJ t          |j        j        j                  h d
k    sJ t          j
        |ddg|          }t          |j        j        j                  h d
k    sJ d|j        vsJ t#          ||                                           t          j
        |d|          }t          |j        j                  h d
k    sJ d|j        vsJ t          j
        |d|          }t          |j        j                  h d
k    sJ d S )Nr  r  rx   r^  rI   r   r(  r  r   >   r'   r(   r  r   r   r   )r   r   r   r   r   r  r   r   r   r   r   rV   r   rc  rb  r(  r   r   r   r/   r  s        r"   test_partition_on_cats_2r  q  s   
f++C
##!!///!;;!!///!;;	
 	
	 	A 	q!ALLC:fL===	V	,	,	,Brtx"##6666rtx"##6666	sCj	@	@	@Brtx"##6666bj    b"**,,	C	7	7	7Brx"##6666bj    	c&	9	9	9Brv !!___444444r$   c           	     X   t          |                     d                    }t          j        t	          j        dt          j                  t	          j        dt          j                  t	          j        dt          j                  t          j	        
                    g dd                              d          d          }t          |          dz  }t          j        |j        d	|         d
          }t          j        |j        |d	         d
          }|                    |||           t#          j        t&                    5 }|                    |dd|           d	d	d	           n# 1 swxY w Y   dt          |j                  v sJ t          |                     d                    }|                    |d||           |                    |dd|           t          j        |d|          }	t-          |                    d          |	           d	S )z#Test append with write_index=False.ztmp1.parquetrq   rr   rt   rx   rz   r{   rI   Nr_  r   rk  FT)r   rZ   r=   Appended columnsztmp2.parquetr   r~   r   )r   r   r   r   r   r   r   r   r   r   r   r   rU   r   r   rn  r   rM   r   rW   r}  r   r   r   )
r   r=   rh  r   r   rp  rq  r   excinfort  s
             r"   test_append_wo_indexr    sC    fkk.))
*
*C	9T2229T2224rz222i&&'@'@'@t&LLSS 		
 	
	
 	
B r77a<D>"'%4%.C888D>"'$%%.C888DOOCMOJJJ	z	"	" LgtFKKKL L L L L L L L L L L L L L LW]!3!33333
fkk.))
*
*COOv=     	OOCU4OGGG?3c&999Dbll3&&&&&s   E99E= E=)r/   offsetz
2022-01-01z
2022-01-31D)freqrF   )daysr   i  c                d   t          |           }t          j        t          j        t          |          t          j                  t          j        t          |          t          j                  t          j        t          |          t          j                  t          j	        
                    g dt          |                                        d          d|          }t          j        |d          }t          j        |                    |j        |z             d          }|                    |||	           t#          j        t&          d
          5  |                    ||d           ddd           n# 1 swxY w Y   |                    ||dd           dS )z1Test raising of error when divisions overlapping.rr   rt   rx   rz   r{   r.   r_  r   rk  z)overlap with previously written divisionsrB  Tr=   rZ   N)r=   rZ   r  )r   r   r   r   r   rU   r   r   r   r   r   r   r   r   r   r/   r   rM   r   rW   )	r   r=   rh  r/   r  r   r   rq  r   s	            r"   !test_append_overlapping_divisionsr    s    f++C	9SZZrx8889SZZrx8883u::RZ888i&&)))E

 '  fSkk	
 	
 

 

 

B >",,,D>",,rx&'899SIIIDOOCMOJJJ	z)T	U	U	U 9 9F48889 9 9 9 9 9 9 9 9 9 9 9 9 9 9 	OOCtdOKKKKKs   'FFFc                   t          |           }t          j        t          j        d          t          j        dd          dt          j        ddd                    }t          j        |dd	          }t          j        t          j        dd          t          j        dd
          d          }|j                            |j        j	                  |_        t          j        |d          }|
                    ||d           |
                    ||d           t          j        ||          }t          j        ||g          }t          ||           d S )Nr_     r&   r   r.   r  F)r1   sort,  r0   Trk  r  r   )r   r   r   r   r   r   r   r/   r   rs   r   r   r  r   )	r   r=   r   r  rq  r   r   ressols	            r"   6test_append_known_divisions_to_unknown_divisions_worksr    s:   
f++C
,inn29S##6#677ryaQS?T?T  C >#15999D
,RYsC00ryc7J7JKK
L
LC 	  11CI>#1---D 	OOCDOAAAOOCtO444
/#f
-
-
-C
)S#J

Cc3r$   c                   t          |           }t          j        dt          j        dt          j                  i          }t          j        dt          j        dt          j                  i          }t          j        dt          j        dt          j                  i          }t          j        |d          }t          j        |d          }t          j        |d          }	|	                    |||           t          j        t                    5 }
|	                    ||d	           d
d
d
           n# 1 swxY w Y   dt          |
j                  v sJ t          j        t                    5 }
|		                    ||d	           d
d
d
           n# 1 swxY w Y   dt          |
j                  v sJ d
S )z-Test raising of error when non equal columns.r|   r_  rr   r}   rI   r   rk  Tr  Nr  zAppended dtypes)r   r   r   r   r   r   r   r   r   r   rM   r   rW   r}  )r   r=   rh  r   r  r   df3rq  r   rt  r  s              r"   test_append_different_columnsr    s    f++C
,ryBH===>
?
?C
,ryBH===>
?
?C
,ryBH===>
?
?C>#+++D>#+++D>#+++DOOCMOJJJ	z	"	" 9gF48889 9 9 9 9 9 9 9 9 9 9 9 9 9 9W]!3!33333	z	"	" 9gF48889 9 9 9 9 9 9 9 9 9 9 9 9 9 9GM 2 2222222s$   EEE F%%F),F)c           	        |dk    rt          j        d           t          |           }t          j        dd          }t          j        dd t          t          |                    D             i|          }t          j	        |d	          }dt          j        d
t          j                    fg          i}|                    |d||           |                    |d||d           t          j        ||          }t          j        ||g          }|                                }	t#          ||	           d S )Nr2   z1Fastparquet engine is missing dict-column supportz
2020-01-01
2021-01-01r}  c                    g | ]}d |iS r  r   )r    r'   s     r"   r#   z+test_append_dict_column.<locals>.<listcomp>  s    555C8555r$   r.   rF   r0   r'   T)rZ   r=   schema)rZ   r=   r  r  r   )rM   rG   r   r   
date_ranger   rangerU   r   r   r  structr   r   r   r  r   r   )
r   r=   r   dtsr   rq  r  r   expectrZ  s
             r"   test_append_dict_columnr    s@    HIII
f++C
-l
3
3C		55U3s88__5556
 
 
B >"!,,,Dry3

"3!4556F 	OOCVFOCCCOOD    
 ?3v...D YBx  F\\^^Fffr$   c                B   t          |           }t          j        g dg dg ddt          j        g dd          g d	          }t	          j        |d
          }t	          j        |||           t	          j        |d|          }t          ||d           d S )Nr  )r*   ry     )r_  r  r  r^  )r  r+   r,   r(  r   r   )r/   r   rI   r0   r   r   Fr   )	r   r   r   Indexr   r   r   r   r   rf  s          r"   test_orderingr  &  s    
f++C	iilllAAh|||)444
 
 
B
 .
+
+
+CM#s<0000?3iDDDDc4//////r$   c                   t          |           }t          j        t          j        dt          j                  t          j        dt          j                  d          }t          j        |d          }|	                    ||           t          j
        |ddg|d	
          }t          |ddg         |d           t          j        t          j                            |d                    }t          j
        |dg|                                          }|                    dd	           t          |dg         |dd           t          j
        |ddg|d	
          }t          |ddg         |d           d S )Nrq   rr   r|   r~   r  r   r   r|   r~   Tr   Fr  r   r   )inplacer   )r   r   r   r   r   r   r   r   r   r   r   r   globr   r   r   r   r  )r   r=   r   r   r   r   fnsr  s           r"    test_read_parquet_custom_columnsr  5  sy   
f++C<	$bh///biBJ6W6W6WXX D 
	+	+	+BMM#fM%%%
/eS\&d  C b%7777
)BGLLk22
3
3C
/#wv
>
>
>
F
F
H
HCOOE4O(((b%k3E5IIII
/c5\&d  C b#u777777r$   zdf,write_kwargs,read_kwargsr'   )r  rI   rF   r  )ccr   bbb)   a   b   cbytes)object_encodingr  )r   r   r   rb  )rF   rI   rF   )i    rq   M8[ns]M8[us]M8[ms]zdatetime64[ns]datetime64[ns, UTC]zdatetime64[ns, CET]uint16float32)r  rF   rI   )r  rF   r   r  foor(   r  )r  rI   N-)      @       @N. c                   d|v r)|j         j        dk    rd|v rt          j        d           d|v rC|j         j        dk    r3|dk    r-t          t          d          k    rt          j        d           t          rrd|v rn|j         j        d	k    s|j         j        d
k    rN|dk    rt          j        d           n3|dk    r-t          t          d          k    rt          j        d           |                    dd           r2|dk    r,t          t          d          k    rt          j        d           t          |           }|j	        j
        d|j	        _
        t          j        |d          }|                    dd           }|r|dk    rt          j        ||f||d| nt          j        ||fd|i| t          j        |f|j	        j
        |dd|}t          |j                            d                    dk    r-|dk    r't#          |                    d          |d           d S t#          ||d           d S )Nr'   r  rg   z7Parquet pyarrow v1 doesn't support nanosecond precisionr   r2   z0.6.3z4fastparquet doesn't support nanosecond precision yetr  r  r5   ,https://github.com/apache/arrow/issues/15079z	2022.12.0z.https://github.com/dask/fastparquet/issues/837rb  z.https://github.com/dask/fastparquet/issues/577r/   rI   r0   r  )r=   r  r=   Tr   UInt16Fr   )r'   rs   rM   rG   fastparquet_versionparse_versionr   rW  r   r/   r-   r   r   popr   r   dtypesr   r   )	r   r   write_kwargsread_kwargsr=   r   r   oer   s	            r"   test_roundtripr  M  s   P byyRTZ8++60A0AUVVVVr		DJ(""m##=#9#999RSSSS 	
R2IITZ8##rtzX'='=YLGHHHH}$$)<A
 A
 *
 *
 L PQQQQ 	d++Gm##=#9#999EFFF
f++C	x}
.
+
+
+C			+T	2	2B	 ?f%%
c3RvrRR\RRRR
c3>>v>>>>?8=T MX D 4;??3  H,,=1H1H 	#**X&&eDDDDDD#tU333333r$   z12.0.0zPKnown failure with pyarrow strings: https://github.com/apache/arrow/issues/33727c                ,   t          |           }t          j        g dt          d          d          }t          j        }|dk    rt          j        j        } |ddi          5  t          j
        |d          }|j                            d	          |d
<   d d d            n# 1 swxY w Y   |                    ||           t          j        |d
g|d          }t          j        ||d          }t          ||           t!          j        t$                    5  |j        j        j         d d d            n# 1 swxY w Y   t          |j                                        j        j                  h dk    sJ |                    d                                           }|                                g dk    sJ |dk    rwt          |j        |j        d           t!          j        t0                    5  t          j        |dg|                                           d d d            n# 1 swxY w Y   t!          j        t2          t4          f          5  t          j        |dg|           d d d            d S # 1 swxY w Y   d S )N)rF   rI   r  r   r   caaabr&   r2   dataframe.convert-stringFrI   r0   r`  r(   r   T)rb  r=   r   r  >   r   r   r(  c                H    | j         j        j                                        S r8   )r(   rc  rb  r  r  s    r"   <lambda>z!test_categories.<locals>.<lambda>  s    QSW-?-K-K-M-M r$   )r   r(  r   r   )check_namesr'   ra  r  )r   r   r   r   
contextlibnullcontextr  rH  rV   r   r   r(   r   r   r   r   rM   r   NotImplementedErrorrc  rb  r   map_partitionsrd  	TypeErrorrW   rK  )	r   r=   r   r   ctxr   r   rt  cats_sets	            r"   test_categoriesr    s}   
 
VB	OOO$w--@@	A	AB

 Cko	(%0	1	1 , ,nRQ///5<<
++C, , , , , , , , , , , , , , , NN2fN%%%?
uV  D
 ?2f$GGGDdD	*	+	+  
              tv~~#.//???BBBB""#M#MNNVVXXH?? 4 4 44444#%U3333]9%% 	K 	KOBC5@@@HHJJJ	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 

M2	3	3 ? ?
wv>>>>? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ?sH   &4B&&B*-B*!D??EE+H88H<?H<#J		JJc                N   t          |           } t          j        g dg ddg d          }t          j        |d          }|                    dg	          }|                    | |
           t          j        | |
          }t          |j	        |j	        d           d S )Nr  )r   r   r   AB)r'   r(   r(   )r   r/   rF   r0   r  r  r   Fr   )
r   r   r   r   r   
categorizer   r   r   r/   r   r=   r   r   r   s        r"   test_categories_unnamed_indexr    s    
 [[F	99???33???
 
 
B .
+
+
+C
..#.
'
'CNN6&N)))?6&111DciU;;;;;;r$   c                   t          |           }t          j        t          d          t          d          d          }t	          j        |d          }||j        dk             }|                    ||           t	          j        ||d          }|j	        dk     sJ |
                                }t          ||dd	           d S )
Nr*   r   r   r0   r   Tr  F)r  r   )r   r   r   r  r   r   r   r   r   r1   r   r   )r   r=   r   r   r   r   rt  r  s           r"   test_empty_partitionr    s    	VB	E"IIE"II66	7	7B
.
+
+
+Csuz?DOOBvO&&& ?2f$GGGDa
,,..Cc4U>>>>>>r$   write_metadatac                "   t          |           }t          j                                        }d|j        _        t          j        |d          }|                    |||           t          j        ||d          }t          ||           d S )Nr  r   r0   rk  Tr  )
r   r   _compatmakeTimeDataFramer/   r-   r   r   r   r   )r   r=   r  r   r   r   r   s          r"   test_timestamp_indexr    s    	VB		%	%	'	'BBHM
.
+
+
+CNN2f.NIII?2f$GGGDc4r$   c                P   t          |                     d                    }t          j        ddt          j        dt          j        dgi          }t          j        |d          }|                    |d	           t          j
        |          }|d         j        dk    sJ d S )
Nr   c1      ?rI   r  rF   r0   r2   r   )r   r   r   r   r   r  r   r   r   r  
read_table
null_count)r   r   r   r   r;  s        r"   0test_to_parquet_fastparquet_default_writes_nullsr    s     
V[[((	)	)B	tc261bfa89	:	:B
.
+
+
+CNN2mN,,,M"E8!######r$   c           
     h   g dddgt           j        t           j        g}g dddgd d g}ddt           j        t           j        g}g d}t          j        dd	          }||t          j        t          j        g}|                                |                                t          j        d
          t          j        d
          g}d}t          j        dd|          }	|	|	t          j        t          j        g}
|	                    d                                           |	                    d                                           t          j        d
          t          j        d
          g}t          j        g d||||
d          }t          j
        |d          }t          j        dt          j        t          j                              fdt          j                    fdt          j        d          fdt          j        d|          fdt          j                    fg          }|                    t%          |           dd|           t          j        t%          |           dd                                                              d          }|j        j        }t1          t3          |                    D ]@}t          j        ||         ||                   sJ t7          ||                               A|j        j        }t1          t3          |                    D ]M}t          j        ||                   rt          j        ||                   sJ 9||         ||         k    sJ N|j        j        }t1          t3          |                    D ]M}t          j        ||                   rt          j        ||                   sJ 9||         ||         k    sJ Nt          j        |j        j        |          sJ t          j        |j         |j                   sJ d S )N)r   rF   rI   r  r   r   r   )r   r   NNiˌ4Zs)unitNaTz
US/Eastern)r  tz)r   r   rF   rF   )partition_columnarraysstringststamps
tz_tstampsrI   r0   r  r  r  nsr  r  r5   )r=   r  r  Fr  Tr   )!r   r  r   	Timestampr  to_datetime64
datetime64
tz_convertr   r   r   r  r  list_r   r)  	timestampr   r   r   r   r   r  valuesr  rU   array_equaltyper  isnatr  r  r  )r   	in_arrays
out_arrays
in_stringsout_stringststamp
in_tstampsout_tstampstimezone	tz_tstampin_tz_tstampsout_tz_tstampsr   r   r  ddf_after_writearrays_after_writer!   tstamps_after_writetz_tstamps_after_writes                       r"   Stest_to_parquet_pyarrow_w_inconsistent_schema_by_partition_succeeds_w_manual_schemar1    s    QFBFBF3I))aVT40JsBFBF+J(((K\*3///F&"&"&1J 	
e
eK HZch???I	2626:M 	T""0022T""0022
e
eN 
 ,!!'	
 	

 
B .
+
+
+CYrx

++,	$T**+2<h778,	
 F NNFI4Fv     	FI5QQQ		$		  )/63r77^^ Y Y~03Z]CCXXT*UV-EXEXXXXX *183r77^^ < <8'*++ 	<8KN++++++&q)[^;;;;; -7>3r77^^ B B8*1-.. 	B8N1-......)!,q0AAAAAA >/18+FFFFF >/:B<OPPPPPPPr$   r  infercomplexc                    |dk    r't          j                    t          j                    d}t          |           } t	          j        g dt	          j        g d          g ddt          dd          	          }|r*t          j	        |d
          
                    d          }nt          j	        |d
          }|                    | d|           t          j        | dd          }t          ||           d S )Nr3  )r/   amount)123r7  r8  r6  4)
2017-01-01r:  r:  
2017-01-02r;  z
2017-01-06z
2017-01-09)r_  r  r    r   X  i  )r/   dater5  r      r.   rI   r0   r/   r5   r=   r  Tr  )r  r)  r   r   r   r   to_datetimer  r   r   r   r   r   r   )r   r/   r  r   df_outs        r"   test_pyarrow_schema_inferencerC  d  s    9;;"(**==[[F	888N  
 
 :99	
 	
 Arll!
 
 
B$  /^BA...88AA^BA...MM&6M:::_VI4PPPFb&r$   c                *   t          j        g dg dd          }t          j        g dg dd          }t          j        t	          j        |          t	          j        |          g|d          }t          j        t                    5 }|	                    t          |           d	           d d d            n# 1 swxY w Y   t          |j                  }d
|v sJ dt          |j                  v sJ dt          |j                  v sJ d S )Nr  g      @r   r   r&   r   r^  Fr  verify_metar5   r   z6Failed to convert partition to expected pyarrow schemaz	y: doublez	y: string)r   r   r   from_delayedr  rE  rM   r   rW   r   r   r}  )r   r  r   r   recmsgs         r"   "test_pyarrow_schema_mismatch_errorrK    sN   
,YYY[[[99
:
:C
,YYY___==
>
>C
/	c		DL--.Se  C 
z	"	" 6cs6{{95556 6 6 6 6 6 6 6 6 6 6 6 6 6 6 ci..CCsJJJJ#ci..((((#ci..((((((s   %B>>CCc                   t          j        g dg dd          }t          j        g dg dd          }t          j        t	          j        |          t	          j        |          g|d          }|                    t          |           dd 	           t          j        | d
          }t          j	        ||g          }t          ||d           d S )Nr  rE  r&   r   r^  FrF  r5   r@  r   )check_dtype)r   r   r   rH  r  rE  r   r   r   r  r   )r   r  r   r   r  r  s         r"   1test_pyarrow_schema_mismatch_explicit_schema_nonerN    s    
,YYY[[[99
:
:C
,YYY___==
>
>C
/	c		DL--.Se  C NN3v;;yN>>>
/&
3
3
3C
)S#J

C c3E******r$   c           
         t          |           } t          j        t          j                            g dd          t          j                            g dd          t          j                            d          t          j                            ddd          t          j        dd          d          }t          j	        |d	
          }|
                    | ddg|           t          j        | |dd                                          }|j                                        D ]L}t          |j        |j        |k                       t          |j        |j        |k                       k    sJ Mt          j        | |ddg                                          }|j                                        D ]L}t          |j        |j        |k                       t          |j        |j        |k                       k    sJ Md S )Nr  r  Cr_  rx   XYZrF   r   r   )a1a2r   r(  r  rI   r0   rV  rW  r  Fr   r  )r=   r   )r   r   r   r   r   r   randintr   r   r   r   r   r   rV  uniquerV   r  rW  )r   r=   r   r  r   r_   s         r"   test_partition_onrZ    s   [[F	)""???"==)""???"==!!s!++""1ac"221c""	
 	

 
B 	rq)))ALLtTl6LBBB /vU  gii  u||~~ D D24%&&#ceCFcM.B*C*CCCCCC /&#t
E
E
E
M
M
O
OCu||~~ D D24%&&#ceCFcM.B*C*CCCCCCD Dr$   c                x   t          |           } t          j        t          j                            g dd          t          j                            g dd          t          j                            d          d          }t          j        |d          }t          d          D ]}|	                    | dd	g|
           t          j
        | |                                          }t          |          t          |          k    sJ t          j        |           D ]\  }}}|D ]}|dv sJ 	d S )NrP  r_  rx   rR  )rV  rW  r   rI   r0   rV  rW  r  r   )r   zpart.1.parquetr   r   )r   r   r   r   r   r   r   r   r  r   r   r   rU   r   walk)r   r=   r   r  rE   r   r   files           r"   test_partition_on_duplicatesr^    s[   [[F	)""???"==)""???"==I$$#$..	
 	

 
B 	rq)))A1XX G G	V4,vFFFF
/&
0
0
0
8
8
:
:Cr77c#hhwv  1e 	 	D      	 r$   r  aac           
     "   t          |           } t          j                            d          5  t          |           } t	          j        t          j                            g dd          t          j                            d          t          j        	                    ddd          d          }t          j        |d	
          }|                    | |dd           t          j        | ddd          }d d d            n# 1 swxY w Y   |                                }|j                                        D ]L}t          |j        |j        |k                       t          |j        |j        |k                       k    sJ Md S )Nzsingle-threadedr   rP  r_  rx   rF   r   )r_  bbr  rI   r0   Fr5   )r  r   r=   )r/   r   r=   )r   r  rH  rV   r   r   r   r   r   rX  r   r   r   r   r   r_  rY  ra  )r   r  r   r  r   r_   s         r"   test_partition_on_stringrb    s    [[F	#4	5	5 
 
V\i&&S&AAi&&C&00i''13'77 
 
 N21---	5 	 	
 	
 	
 o%U9
 
 

 
 
 
 
 
 
 
 
 
 
 
 
 
 
  ++--Cu||~~ F F25#&''3svcfm/D+E+EEEEEEF Fs   CD

DDc                h   t          |           } g d}t          j        g dt          j        ||d          d          }t	          j        |d                              d          }|                    | d	|
           t	          j        | d|dgd          }t          |          dk    sJ d S )N)z
2018-01-01
2018-01-02z
2018-01-03z
2018-01-04)rF   rF   rF   rF   T)rb  ordered)dummyDatePartr   r0   rf  rg  r  )rg  <=rd  )r/   r=   filtersr   rI   )
r   r   r   Categoricalr   r   r   r   r   rU   )r   rJ   rK   catsdftestddftestddftest_reads          r"   test_filters_categoricalro    s    [[FCCCD\!\\tdKKK	
 	
 F nV333==gFFGvJ|LLL?24   L |!!!!!!r$   c                   t          |           }t          j        t          d          t	          d          d          }t          j        |d          }|j        dk    sJ |                    ||d           t          j	        ||dg	          }|j        d
k    sJ |j
        d
k                                                                    sJ t          j	        ||dg	          }|j        dk    sJ |j        dk                                                                    sJ t          j	        ||ddg	          }|j        dk    sJ t          |          rJ t          ||           t          j	        ||ddgddgg	          }	|	j        d
k    sJ |	j
        dk    |	j
        dk     z                                                                  sJ t          j	        ||dg	          }
|
j        dk    sJ |
j
        dk     |
j
        dk    z                                                                  sJ t          j	        ||dg	          }|j        dk    sJ t          |          sJ |j        dk                                                                    sJ t          j	        ||dg	          }|j        dk    sJ d S )Nr*   
aabbccddeer&   r   r0   Trk  )r'   >r   r=   ri  r  )r(   ==r(  rF   r(  )r'   rr  r   )r'   rr  rF   )r'   <r   r'   rr  r  )r'   ru     rw  )r'   in)r   	   rI   r   )r(   =r(  )r'   !=rF   )r   r   r   r  r   r   r   r1   r   r   r'   r   r   r(   rU   r   )r   rJ   rK   r   r   r   r   r   r(  r  er~   gs                r"   test_filtersr~    s   6{{H	E"IID,>,>??	@	@B
.
+
+
+C?aNN8LdNKKK
}oNNNA=AC!G==??""$$$$$
?O>PQQQA=AC3J%%'''''
/?.O	 	 	A =A1vvaOOO
 M*M*
	 	 	A =AS1Wq!&&((0022222
?R>STTTA=AS1Wq!&&((0022222
>OPPPA=Aq66MMMC3J%%'''''
~>NOOOA=Ar$   c                <   |dk    s|dk    rt          j        dd           |dk    }t          |           }t          j        dg di          }t          j        |d          }|                    dd	
                              |d|           t          j	        |d|dg          
                                }t          j	        |d|dg          
                                }|rEt          |||d         dk             d           t          |||d         dk             d           n t          ||           t          ||           |                    dd	
                              ||           t          j	        ||          
                                }t          ||           |dk    r|                    dd	
                              ||           t          j        |                              dg          }	t          j        |                              dg          }
t          |	          dk    sJ t          |
          dk    sJ |                    dd	
                              ||           t          j	        ||dg          
                                }t          j	        ||dg          
                                }t          |          dk    sJ t          |          dk    sJ t          ||           d S )Nr2   0.3.1
minversionr5   at)abr_  badara  rF   r0   T)r1   forceFr   )r  rt  r_  r/   r=   ri  )r  rz  r_  r_  r  rI   r   ri  r   rs  )rM   importorskipr   r   r   r   r   repartitionr   r   r   r   r2   ParquetFile	to_pandasrU   )r   rJ   rK   pyarrow_row_filteringr   r   r   r   rt  r   r  s              r"   test_filters_v0r  @  sE   }$$}(D(DMg>>>> (94	VB	t;;;<	=	=B
.
+
+
+C OOO..99
l :    ?
%6H5I  gii 	 ?
%6G5H  gii 	  $CI-.EBBBB$CI-.EBBBBB$$ OOO..99"\9RRR?2k222::<<DdC m##AT22==b=VVV%b))33=O<P3QQ%b))33=N<O3PP3xx!||||3xx!|||| OOO..99"\9RRR?
;);(<  gii 	 ?
;):(;  gii 	 t99q====t99q====dDr$   c                   t          j        dd           t          |           }t          j        t          d          ddgdz  d          }t          j        |d	
          }|                    |d|d           d}d}dd|fdd|fgg}t          j	        |dd|          }d}	||d         |k              j
        D ]7}
|	t          t          |
                                          dk              z  }	8|j        |	k    sJ ||d         |k              }||d         |k             }t          ||                                d           d S )Nr5   z1.0.0r  r_  rc  dogr  )r_  ra  r*   r0   FTr   r   r_  ru  ra  rt  r  r   r  )rM   r  r   r   r   r  r   r   r   r   rN  intrU   r   r1   r   )r   r=   r   r   r   aa_limbb_valri  r   nonemptyparts              r"   test_filtering_pyarrow_datasetr  x  se   
	g6666	VB	U3ZZu~/BCC	D	DB
.
,
,
,CNN25TNRRR FFsF#dD&%9:;G?2U9gNNND HCI&'2 1 1CDLLNN++a/000x'''' 
BtHv	B	BtH	Bb$,,..e444444r$   c                >   t          j        t          d          t          d          d          }t	          j        |d          }|                    t          |           |           t          |                     d                    }t	          j	        |d|d	g
          }|j
        dk    sJ t          ||d         dk             |                                d           t	          j	        t          |                     d                    d|d	g
          }t          |          dk    sJ t          j	        t          j                            | d          |          t!          |j                                               t          j                            | d          |           t	          j	        t          |                     d                    d|d	g
          }|j
        dk    sJ t          ||d         dk             |d           d S )Nr*   rq  r&   r   r0   r   r   Trv  )r   r=   ri  r  r'   Fr  r   r   zpart.4.parquet)r   r   r  r   r   r   r   r   r   r   r1   r   r   rU   r   r   reversedr   )r   r=   r   r   r   ddf_outr   rt  s           r"   test_filters_file_listr    s   	E"IID,>,>??	@	@B
.
+
+
+CNN3v;;vN...K(())Eo4  G !####bC1ow00eDDDD ?FKK())** 	  D t99>>>> OBGLL)9::6JJJjf&677jGGG?FKK$$%% 	  D q    bC1ot777777r$   c                   t          j        d           t          j        g dt	          d          d                              d          }|j        d d                             t          | 	                    d                    dd	           |j        dd                              t          | 	                    d
                    dd	           t          j        t          |           ddddg          }|j        dk    sJ t          j        t          |           ddddg          }|j        dk    sJ d S )Nr5   )r   rF   r*   rU  rI   r  rw  ry  rw  r   r   r   zfile.0.parquetrI   r=   row_group_sizezfile.1.parquetFT)r   rh  r  )r=   split_row_groupsr   ri  r   rI   r  )rM   r  r   r   r  r   rn  r   r   r   r   r   r   )r   r   r   s      r"   test_pyarrow_filter_divisionsr    s^   
	""" 
666U1XXFF	G	G	Q	QRU	V	VBGBQBKFKK())**9Q     GABBKFKK())**9Q     /F    C =I%%%%
/F    C =I%%%%%%r$   c           	        t          j        dd           t          |           } d}g }dD ]}||gt          |dz            z  z  }t	          j        |t          j                            |          t          j                            dd	|          d
          }t          j
        |d          }|                    | ddgd           t          j        | ddgd          }d}|j        |k    sJ d S )Nr2   r  r  r_  r*  r   rx   rF   r   r^  r0   Tr   r   r  r=   )r   rt  r   r=   ri  r   )   1   )rM   r  r   r  r   r   r   r   rX  r   r   r   r   r   )r   ry   categoricalsr}  r   r  r   expected_divisionss           r"    test_divisions_read_with_filtersr    s    
'::::[[FDL% 0 0#dQh--//	!!t!,,""1ad"33	
 	

 
B 	rq)))ALLTmLTTT
/!" 	  C "=.......r$   c                N   t          j        dd           t          |           } t          j        g dg ddg d          }t          j        |d	          }|                    | d
gd           t          j        | ddgd          }|j	        sJ d}|j
        |k    sJ d S )Nr2   r  r  )r   r   rF   rF   rI   rI   r  r  )id1id2r  r  r  r  r  r  )rY  idr.   rI   r0   r  r  )r  rt  r  Tr  r  )rM   r  r   r   r   r   r   r   r   r   r   )r   r   r  r   r  s        r"   *test_divisions_are_known_read_with_filtersr    s    
'::::[[F	...JJJ	
 	
 '&&
 
 
B 	rq)))ALLtf]LCCC
/$% 	  C "=.......r$   z$No longer accept ParquetFile objectsc           	        t          |           }t          j        t          j                            g dd          t          j                            d          t          j                            ddd          d          }t          j        |d          }|	                    |d	gd
           t          j        |          }t          j        |                                          }|j                                        D ]L}t!          |j        |j        |k                       t!          |j        |j        |k                       k    sJ Mt          j        |dg                                          }t!          |j        |j        dk                       t!          |j                  k    sJ t%          j        t(                    5  t          j        |d          }d d d            d S # 1 swxY w Y   d S )NrP  r_  rx   rF   r   r^  rI   r0   r   r2   r  )r   rt  r  r  r  r5   r   )r   r   r   r   r   r   rX  r   r   r   r2   r  r   r   r   rY  rV   r   rM   r   rI  )r   r   r   r  pq_fr   r_   s          r"   &test_read_from_fastparquet_parquetfiler    s
    
VB	!!///!<<!!s!++""1ac"22	
 	

 
B 	rq)))ALL3%L>>>"2&&D /$


'
'
)
)Ct{{}} B B24$%%SU35C<-@)A)AAAAAA /$)9(:
;
;
;
C
C
E
ECrtBDCK !!SZZ//// 
~	&	& 6 6od95556 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6s   G77G;>G;r   threads	processesc                   t          |           } t          j        g dg dd          }d|j        _        t          j        |d          }|                    | d|          }t          |d	          sJ |	                    |
           t          j                            |           sJ t          j        | |d          }t          ||d           d S )NrF   rI   r  r   )r	  r  r  g      @r   r/   rI   r0   Fr   r=   r  r   Tr  r   )r   r   r   r/   r-   r   r   r   hasattrr   r   r   r   r   r   )r   r   r=   r   r   r}  r   s          r"   test_to_parquet_lazyr  5  s    [[F	LLL/C/C/CDD	E	EBBHM
.
+
+
+CNN65N@@E5&!!!!!	MMIM&&&7>>&!!!!!?6&dKKKDc4//////r$   r   c                   ddl m} t                      }|                    |d|           t                              | |d           |                    t          |                     }|j        sJ |j	        j
        d         |k    sJ d S )Nr   LocalFileSysteminvalidate_cacher5   r  )fsspec.implementations.localr  r   setattrr   r   _strip_protocolr   called	call_argsargs)r   monkeypatchr   r  r  r   s         r"   &test_to_parquet_calls_invalidate_cacher  F  s     =<<<<< {{);=MNNNNN679N===**3v;;77D""""%*1-555555r$   c                   t          |           }t          j        dt          j        dd          gi          }t	          j        |d          }|                    |ddd	           t          j        |          }|j	        d         j
        t          j        j        j        k    sJ t	          j        |dd
                                          }t!          ||           d S )Nr   nowTutcrF   r2   Fint96)r=   r   timesr   )r   r   r   rA  r   r   r   r2   r  _schemar   parquet_thriftTypeINT96r   r   r   )r   r   r   r   pfr   s         r"   test_timestamp96r  S  s    	VB	sR^Et<<<=>	?	?B
.Q

CNN2mgNNNN		 	$	$B:a=!;!@!FFFFF
/"]%
@
@
@
H
H
J
JCc2r$   c                H  	 t          |           }d	t          j        	fdt          g d          D                       }t          j        	fdt          g d          D                       }g }dD ]}t          j                            ||          }t          j                            |          st	          j        |           |	                    t          j                            |d                     t          j        |d         |           t          j        |d         |           t          j        |d	
          }d|j        v sJ |                                }d|v sJ t!          j        |j                  ddgk                                    sJ d S )Nr   c                V    i | ]%\  }}|t           j                                      &S r   r   r   r    r!   r(  Ns      r"   rD   z%test_drill_scheme.<locals>.<dictcomp>c  /    VVV41a29++A..VVVr$   r^  c                V    i | ]%\  }}|t           j                                      &S r   r  r  s      r"   rD   z%test_drill_scheme.<locals>.<dictcomp>d  r  r$   )
test_data1
test_data2z
data1.parqr   rF   r2   r   dir0r  r  )r   r   r   rG  r   r   r   r   mkdirrZ   r2   r  r   r   r   r   r   rY  r  r   )
r   r   r  r   r   r  dnr   r   r  s
            @r"   test_drill_schemer  _  s   	VB	A
,VVVV9___;U;UVVV
W
WC
,VVVV9___;U;UVVV
W
WCE) 5 5W\\"a  w~~b!! 	HRLLLRW\\"l334444eAh$$$eAh$$$	}	5	5	5BRZ
**,,CS====IchL,#??DDFFFFFFFr$   c                   t          |           }t          j        t          j        t          j                            g dd          d          t          j        t          t          dd                    d          t          j        t          t          dd                    d          d	          }t          j
        |d
          }|                    ||           t          j        |dg|          }t          |j                  dgk    sJ t          j        ||          }t          |j                  t          |          k    sJ d S )Nr   r   r(  r  r|  r~   r_  rx   r`  rr   r   r  float)rb  intsfloatsrF   r   r  r   )r   r   r   r+  r   r   r   r   r  r   r   r   r   r   )r   r=   r   r   r   rddfs         r"   test_parquet_select_catsr  v  s=   	VB	)	  !?!?!?c JJ    Id5C==11???iU1c]] 3 37CCC	
 	
	
 	
B .Q

CNN2fN%%%?2x???D&))))?2f---Db))))))r$   c                   |dk    r,t           t          d          k    rt          j        d           t	          |           }t          j        dddgit          j        ddgd	
                    }d|j        _	        t          j        |d          }|                    ||           t          j        ||d	g          }t          ||           d S )Nr2   r  z5Fastparquet does not write column_indexes up to 0.3.1r  rF   rI   r   r   idxr,   r.   colsr   r   )r  r  rM   rH   r   r   r   r  r   r-   r   r   r   r   r   )r   r=   r   r   r   rZ  s         r"   test_columns_namer    s    #6-:P:P#P#PKLLL6{{H	sQFm28S#JU+K+K+K	L	L	LBBJO
.Q

CNN8FN+++_XfUGDDDFfbr$   c                   | dk    r`t          j        |          }|j        j        d         j        d         j        }||j        |j        k    sJ d S |j        |j        k    sJ d S t          j	        
                    t          j                            |d                    }|j        j        }t!          |j                  D ]}|                    |          }t!          t'          |                    D ]y}	|                    |	          }
||
j        |
j        k    sJ ,|}|dk    rd}|                                |
j                                        k    sJ |
j        |
j        k    sJ zd S )Nr2   r   r   defaultsnappy)r2   r  fmd
row_groupsr   	meta_datatotal_compressed_sizetotal_uncompressed_sizer  parquetread_metadatar   r   r   r  r   r  num_row_groups	row_grouprU   r   lowercompression)r=   filenamer  r  mdmetadatar   r!   r  jr   compress_expects               r"   check_compressionr    s   $X..Vq!)!,6+r/IIIIIII+r/IIIIIII:++BGLL;,O,OPP%x.// 	 	A **1--I3u::&&  "))!,,&48VVVVVV '2O"i//*2*0022f6H6N6N6P6PPPPP48VVVVVV	 	r$   zcompression,)Ngzipr  c                L   t          |           }t          j        g ddz  g ddz  d          }d|j        _        t          j        |d          }|                    |||d	           t          j        ||d
          }t          ||           t          |||           d S )Nr^  r*   r  r&   r/   r  r0   T)r  r=   r   r  )r   r   r   r/   r-   r   r   r   r   r   r  )r   r  r=   r   r   r   r   s          r"   %test_writing_parquet_with_compressionr    s    	VB	OOOb0yyy2~FF	G	GBBHM
.
+
+
+CNN2;vSWNXXX
/"V
F
F
FCc3fb+.....r$   c                   t          |           }t          j        g ddz  g ddz  d          }d|j        _        t          j        |d          }|                    |||dgd	
           t          |||           d S )Nr^  r*   r  r&   r/   r  r0   r'   T)r  r=   r  r   )	r   r   r   r/   r-   r   r   r   r  )r   r  r=   r   r   r   s         r"   6test_writing_parquet_with_partition_on_and_compressionr    s    	VB	OOOb0yyy2~FF	G	GBBHM
.
+
+
+CNN
U      fb+.....r$   r  r   r  r-   
numpy_typepandas_typer  0.21.0r   index_columnspandas_versionencodingUTF-8objectunicode
field_namer  r-   r  r  __index_level_0__column_indexesr   r  r  c                    | j         S r8   r9   r;   s    r"   pandas_metadatar    s    T =r$   c                    t          |           \  }}}}|dgk    sJ |dgk    sJ |d gk    sJ | d         dgk    r|dddk    sJ n|dddk    sJ t          |t                    sJ d S )Nr  r  r  r  r  r  )r  r  )r   
isinstancedict)r  index_namescolumn_namesmappingcolumn_index_namess        r"   test_parse_pandas_metadatar  "  s    =S> >:Kw(: 5'!!!!C5    $'''' ',?+@@@C@@@@@@@%c222222gt$$$$$$$r$   c                 V   d g} dg}d dd}d g}d ddddd ddddgdgdd}t          |          \  }}}}|| k    sJ ||k    sJ ||k    sJ ||k    sJ d dd	id d
ddgdd dddddd d dddgdgdd}t          |          \  }}}}|| k    sJ ||k    sJ ||k    sJ ||k    sJ d S )Nr'   )r  r'   r   r  r  r  r  r	  r
  r  r  r  r  r   )	e_index_namese_column_names	e_mappinge_column_index_namesr  r  r  r  r  s	            r"   %test_parse_pandas_metadata_null_indexr"  3  s   FMUN&*55I 6
 !%&	  !+%&	 
 .."!
 
B$ >TTV=W=W:Kw(:-''''>))))i!55555 #'1&( 
 " %&  2 %& 
  .."7
 
B: >TTV=W=W:Kw(:-''''>))))i!5555555r$   c                p   t          |           dz   }t          j                            t          j        g d          t          j        g d          gddg          }t          j        ||           t          j        ||          }t          j
        g dg dd          }t          ||           d S )	Nz
table.parqr  )r  r   r   r  r  r   r   r  )r   r  r  r   arrayr  r  r   r   r   r   r   )r   r=   r   r;  rZ  r[  s         r"   test_read_no_metadatar%  w  s     f++
$CH  	)))		bhyyy1123* !  E N5#_S000F|))))))<<==Hfhr$   c                     d ddid dddgdd dddddd ddddgdgd	d
} t          |           \  }}}}|dgk    sJ |dgk    sJ |dddk    sJ |d gk    sJ d S Nr	  r
  r  r  r  r  r   r  r  r  r  r   r  r  r  storage_name_mappingr  s        r"   2test_parse_pandas_metadata_duplicate_index_columnsr*         #'1&( 
 " %&  2 &( 
  .."7
 
BD 	r""3%C5    3#G#GGGGG$''''''r$   c                     d ddid dddgdd dddddd ddddgdgd	d
} t          |           \  }}}}|dgk    sJ |dgk    sJ |dddk    sJ |d gk    sJ d S r'  r   r(  s        r"   1test_parse_pandas_metadata_column_with_index_namer-    r+  r$   c           	     2   t          |           }t          j                            |d          }t          j                            |d          }t	          j        t          j                            g dd          t          j                            d          t          j        	                    ddd          d          }d	|j
        _        t          j        |d
          }dd ddddd dd} |j        |fd|i||          t          j        ||d          }t!          |||dk               t"          j                            d          5   |j        |f|dgd||          d d d            n# 1 swxY w Y   t          j        ||                                          }|j                                        D ]L}	t'          |j        |j        |	k                       t'          |j        |j        |	k                       k    sJ Md S )NnormalpartitionedrP  r_  rx   rF   r   r^  r/   r  r0   r  T)r  coerce_timestampsuse_dictionaryr   )r  r  
fixed_text)r5   r2   r=   r  r2   r  r   r   r   r=   r  r   )r   r   r   r   r   r   r   r   r   rX  r/   r-   r   r   r   r   r   r  rH  rV   r   r   rY  r   )
r   r=   r   path1path2r   r   engine_kwargsr   r_   s
             r"    test_writing_parquet_with_kwargsr8    so   	VBGLLX&&EGLL]++E	!!///!<<!!s!++""1ac"22	
 	

 
B BHM
.
+
+
+C $!%"
 

 (0'QUVV M CN5AAA=+@AAA
/%D
I
I
ICc3V}%<>>>> 
6	*	* 
 
	
 u	
 	
8Ef8M	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 /%
/
/
/
7
7
9
9Ct{{}} B B24$%%SU35C<-@)A)AAAAAAB Bs   E>>FFc                    t          |           }t          j        t                    5  t                              ||d           d d d            d S # 1 swxY w Y   d S )Nunknown_value)r=   unknown_key)r   rM   r   r  r   r   r   s      r"   (test_writing_parquet_with_unknown_kwargsr<   	  s    	VB	y	!	! G Gr&oFFFG G G G G G G G G G G G G G G G G Gs   AAAc                |   ddl m t          |           } dgfd}t          j        g dg dd          }t          j        |d	          }|                    | |d
|i           d         sJ t          j        t          j
                            | d          |          }t          ||d           d S )Nr   )rW  Fc                     dd<    | i |S )NTr   r   )r  r]   flagmp_gets     r"   my_getz(test_to_parquet_with_get.<locals>.my_get	  s!    Qvt&v&&&r$   r*  r  r&   rI   r0   r   )r=   compute_kwargs*r   r  )dask.multiprocessingrW  r   r   r   r   r   r   r   r   r   r   r   )r   r=   rA  r   r   rZ  r?  r@  s         @@r"   test_to_parquet_with_getrE  	  s    222222[[F7D' ' ' ' ' ' 
000|||DD	E	EB
.
+
+
+CNN6&+v9NNOOO7NNN_RW\\&#66vFFFFfbe,,,,,,r$   c                   t          |           }d}t          j                            dd|                                          dz   t          j                            g d|          t          j                            g d|          d}t          j        t          j	        |          d	          }|
                    |d
d|ddg           t          j        ||          }||j        dk                                              d S )Nry  r   r%  rx   r  rP  )r  EF)signal1fake_categorical1fake_categorical2rI   r  FrJ  rK  )r  r   r=   r  r   r  )r   r   r   r/  cumsumr   r   r   r   r   r   r   rJ  r   )r   r=   r   ry   r  r   df_partitioneds          r"   test_select_partitioned_columnrN  	  s   	VBD9##As#66==??"DY--oooD-IIY--oooD-II	 	A
 
Q	+	+BMM
)+>?     _R777N>3s:;CCEEEEEr$   c                   |dk    r,t           t          d          k     rt          j        d           t	          j                    5  |dk    rt	          j        dt                     t          |           }t          j
        dggdgd          }t          j        |d	          }|                    ||
           t          j        ||
          }t          ||dd           d d d            d S # 1 swxY w Y   d S )Nr2   z0.3.0z&fastparquet<0.3.0 did not support thisignorer   r   r  )r   rs   rF   r   Fr   r   )r  r  rM   rH   warningscatch_warningssimplefilterrK  r   r   r   r   r   r   r   r   )r   r=   r   r   r   s        r"   test_with_tzrU  1	  sP   #6w9O9O#O#O<===		 	"	" 	I 	I]""!(M:::VBsecU:OPPPBA&&BMM"VM,,,/"V444Cb#u%HHHH	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	Is   BC22C69C6c                   t          |           }t          j        t          j        d          d                              t          j                  t          j        dd                              t          j                  t          j        dd                              t          j                  t          j        dd                              t          j                  d}t          j
        |          }t          j        |d          }|                    |d	d
d           t          j        |d
d	          }|                    dt          j        i                                           d S )Nr  rI   r  r   r  r  )r@  r   r(  r  r0   r5   Fr@  )r=   r   r  r   r   )r   r   repeatr   r   int8int16r  r   r   r   r   r   r   r   r   )r   r   r   pdfr   s        r"   test_arrow_partitioningr[  A	  s    v;;DYry||Q''..rw77Yr1$$RX..Yr1$$RZ00Yr1$$RZ00	 D ,t

C
.!
,
,
,CNN4	u3NOOO
/$eI
>
>
>CJJRZ !!))+++++r$   c                 $   t          j        t                    5 } t          j        dd           d d d            n# 1 swxY w Y   dt          | j                  v sJ dt          | j                  v sJ dt          | j                  v sJ d S )Nr  r   rg   r2   )rM   r   rW   r   r   r   r}  )infos    r"   test_informative_error_messagesr^  T	  s    	z	"	" -d
e,,,,- - - - - - - - - - - - - - - C
OO####c$*oo%%%%C
OO++++++s   =AAc                   t          |           }t          j        dg di          }|d                             d          |d<   t	          j        |d          }t	          j        |||           t	          j        ||dd|           t	          j        ||                                          }|d         	                                g dd	z  k    sJ d S )
Nr'   )r   r   r   r   r   r`  rF   r0   r   T)rZ   r  r=   rI   )
r   r   r   r   r   r   r   r   r   rd  )r   r=   r   r   r   r  s         r"   test_append_cat_fpr`  ]	  s    v;;D	s5556	7	7BgnnZ((BsG
.
+
+
+CM#tF++++M#tD4OOOO
V,,,4466AS6==??777!;;;;;;;r$   r   r   r   r   rF   rI   r  )i- i @B 
13.0.0.devr  rF   rI   r  r   r   r   c                    t          |           }|j        j        sd|j        _        t          j        |d          }t          j        ||dd           t          j        |dd          }t          ||           d S )Nr/   rI   r0   r5   Tr   r  )r   r/   r-   r   r   r   r   r   )r   r   r   r   r   s        r"   test_roundtrip_arrowrf  k	  s~    X 6{{H8=  
.
+
+
+CM#x	tDDDD?8I4PPPDc4r$   c                   t          |           }t          j                            ddd                                          }|                    ||           t          j        ||d          }t          ||           d S )N
2000-01-01z
2000-01-101dstartendr  r   Tr  )	r   r  datasets
timeseriespersistr   r   r   r   )r   r=   r   r   r   s        r"   test_datasets_timeseriesrp  	  s    6{{H		!	!4 
" 
 
gii  MM(6M***
/(6t
L
L
LCb#r$   c                B   dd l }t          j        dg di          }d|j        _        t          j        |d          }|                    t          |                     }|	                    ||           t          j
        ||d	          }t          ||           d S )
Nr   r'   ra  r/   rI   r0   r   Tr  )pathlibr   r   r/   r-   r   r   Pathr   r   r   r   )r   r=   rr  r   r   r   r   s          r"   test_pathlib_pathrt  	  s    NNN	s.../	0	0BBHM
.
+
+
+C<<F$$DNN4N'''?4DIIIDc4r$   c                   t          |                     d                    }t          j                            ddd          }t          j        |j        dg          }|j        	                    d          |_        |
                    |d	d
           t          j        ||ddi          }t          t          |j        j        j                  t          |                                j        j        j                             d S )Nzparquet_int16.parqr   i 5 rb  rx   r-   r  r`  r2   uncompressed)r=   r  i8 )r=   rb  )r   r   r   r   rX  r   r   Tr-   r   r   r   r   r   r\   rc  rb  r   )r   r=   r   numbersr   r   s         r"   test_categories_largery  	  s     
V[[-..	/	/Bi688G	gi&	2	2	2BgnnZ((BGMM"]MGGG
/"V
H
H
HCfRW[+,,fS[[]]5G5K5V.W.WXXXXXr$   c                    t          |           }t                              ||           t          j        t
          j                            |d          |d          }t          t          |d           d S )Nr   r   Fr  r   )	r   r   r   r   r   r   r   r   r   r   s        r"   test_read_glob_no_metar{  	  so    6{{HNN8LN111?
X{++!  D
 c4//////r$   c                   t          |           }t                              ||d           t          j        t          j                            |d                    }|                    t          j                            |d                     t          j	        ||d          }t          t          |d           d S )NTrk  r   r   Fr  r   )r   r   r   r  r   r   r   rZ   r   r   r   )r   rJ   rK   r   pathsr   s         r"   test_read_glob_yes_metar~  	  s    6{{HNN8LdNKKKIbgll8[99::E	LLh44555?5%PPPDc4//////r$   r   remove_commonc                   t          |           }t                              ||d           t          j                            t          j                            |d                    r2t          j        t          j                            |d                     t          j        |          }d|vsJ |rot          j                            t          j                            |d                    r2t          j        t          j                            |d                     t          j
        |||          }t          t          ||           d S )NTrk  r   r   r  r   r   )r   rJ   rK   r   r  r   r   r   s           r"   test_read_dir_nometar  	  s    6{{HNN8LdNKKK	w~~bgll8[99:: 7
	"',,x55666Jx  Ee#### >X?Q(R(RSS >
	"',,x);<<===?8KYWWWDc4333333r$   c                    t          |           }t                              ||d           t          j        ||d          }t          t          |           d S )NFrk  Tr  r   r   s        r"   test_statistics_nometar  	  sQ    6{{HNN8LeNLLL?8KTRRRDc4r$   c                d   t          |                     d                    }t          j                            |dd          }t
          j                            ddd                                          	                    d           }|
                    d                                                                          }|j                            |j        dk    d           |_        |                    ||d	|
           t!          j        ||          }t%          ||d	d	           d S )Nr   z../rh  z
2000-01-031hrj  c                     | j         d d         S )Nr   )re  r  s    r"   r  z1test_timeseries_nulls_in_schema.<locals>.<lambda> 
  s    !%) r$   r'   F)r=   r   r  r   rQ  )r   r  r   r   r   r  rm  rn  r   r  r   ro  r-   wherer  r   r   r   r   )r   r=   r  r   r   ddf_reads         r"   test_timeseries_nulls_in_schemar  	  s    6<<(())Hw||HeW55H 	  |D QQ		++	,	, 	
 >>#**,,4466D	, >EEDI 	OOHVvOVVVx777HheGGGGGGr$   c                @   dd l }t          |           }t          j                            dddd          }|                    ||           t          j        ||          }t          |	                    |
                                                    dk     sJ d S )	Nr   rh  z
2000-01-0260sr  )rk  rl  r  partition_freqr   ia  )pickler   r  rm  rn  r   r   r   rU   dumps__dask_graph__)r   r=   r  r   rq  r   s         r"   test_graph_size_pyarrowr  
  s    MMM	VB=##5 $  D 	OOBvO&&&?2f---Dv||D//112233e;;;;;;r$   r   r  c                   t          |                     d                    }t          |                     d                    }t          j        ddgdz  ddgdz  dd	gdz  d
|          }d|j        _        t          j        |dd          }|                    |||           t          j	        ||          d         }|
                                                    ||d          }t          |j        |j        g          }	t          |	d          }
t          |
t                     sJ |
j        dgk    sJ t%          t'          |
j                                                            d         j        dgk    sJ t          |	d          }t          |t,                    sJ t/          |                    d          |                                           d S )Nr  r  rF   rI   rq   r  r   r   r   rP  r.   my_indexFr  r   r   r  )r=   r   keysread-parquetr   
to-parquetoptimize_graph)r   r  r   r   r/   r-   r   r   r   r   to_framer   r  rb   r   r  r   r   nextiterdskr  r   r   r   )r   r=   r   r/   tmp_path_rdtmp_path_wtr   r   r   r  subgraph_rdsubgraph_wts               r"   test_getitem_optimizationr  
  s    fll6**++Kfll7++,,K	!ftm1a&4-q!ftmDDE
 
 
B BHM
.QU
+
+
+CNN;v>NJJJ
/+f
5
5
5c
:C ,,..
#
#K
#
N
NC
$SXSWI
>
>
>CC00Kk#3444443%''''[_++--..//2:seCCCCC..Kk9-----ckkk//?????r$   c                "   t          j        dgdz  dgdz  dgdz  dgdz  d          }t          j        |dd          }t          j                            t          |                     }|                    ||	           t          j	        ||	          g          }t          |j        |j        g
          }t          d |j                                        D                       }|j        g k    sJ t#          ||g                     d S )NrF   r_  rI   r  r   r  r  rQ  r  Fr  r   r  c              3  D   K   | ]}t          |t                    |V  d S r8   )r  r   )r    ls     r"   r   z2test_getitem_optimization_empty.<locals>.<genexpr>B
  s2      VV!jDT6U6UVAVVVVVVr$   )r   r   r   r   r   r   r   r   r   r   r   r  _namer  layersr  r   r   )r   r=   r   r   r   r   r  subgraphs           r"   test_getitem_optimization_emptyr  9
  s   	QC#IQC#IQC#IQCRUIVV	W	WB
.QU
+
+
+C	c&kk	"	"BNN2fN%%%?2f---b1D
$TYdj\
B
B
BCVVsz0022VVVVVHr!!!!dCGr$   c                   t          j        dgdz  dgdz  dgdz  dgdz  d          }t          j        |d          }t          j                            t          |                     }|                    ||           t          j	        ||          d         }t          j	        ||          d	g         }t          j	        ||          d	d
g         }t          j        |||          \  }}	}
t          j        |||d          \  }}}t          ||           t          |	|           t          |
|           d S )NrF   r_  rI   r  r   r  r   r  rQ  r  Fr  )r   r   r   r   r   r   r   r   r   r   r  r   r   )r   r=   r   r   r   r   r   r(  rV  rW  a3b1b2b3s                 r"   test_getitem_optimization_multir  H
  s4   	QC#IQC#IQC#IQCRUIVV	W	WB
.Q

C	c&kk	"	"BNN2fN%%%
6***3/A
6***C51A
6***C:6AaA&&JBBaAe<<<JBBb"b"b"r$   c                8   t          j        g ddz  t          d          t          d          d          }t          j        |d                              | |           t          j        | |          }||d         d	k             d
g         }||d         d	k             d
g         }t          |j        |j	        g          }t          |d          }t          |t                    sJ t          |j                  d
dhk    sJ t          ||           d S )Nr  r   r   r^  r  r0   r   r   r*   r   r  r  )r   r   r  r   r   r   r   r   r  r  r   r  r   rV   r   r   r   r=   r   r   r   r   r  r  s           r"   &test_getitem_optimization_after_filterr  Z
  s   	IIIMb		b		JJ	K	KBN21%%%000GGG
/&
0
0
0C
RWr\
C5
!Cs3x"}se$D
$TYdj\
B
B
BCC00Kk#344444{"##Sz1111c4r$   c                   t          j        g ddz  t          d          t          d          d          }t          j        |d                              | |           t          j        | |          }|dg         }|                    d	
          }||d         d	k             dg         }|dg         }|                    d	
          }||d         d	k             dg         }t          |j	        |j
        g          }t          |d          }t          |t                    sJ t          |j                  dhk    sJ t!          ||           d S )Nr  r   r   r^  r  r0   r   r   rF   )r  r  r  r  )r   r   r  r   r   r   r   assignr   r  r  r   r  r   rV   r   r   r  s           r"   .test_getitem_optimization_after_filter_complexr  j
  sI   	IIIMb		b		JJ	K	KBN21%%%000GGG
/&
0
0
0C
cU)C
**q*//C
SX]
SE
"Cu:D;;;DtCyA~u%D
$TYdj\
B
B
BCC00Kk#344444{"##u,,,,c4r$   c                   t          j        t          d          ddgdz  d          }t          j        |d                              | |dg	           d
g}t          j        | ||          }d|d                                         vsJ t          j        | |          }t          j	        t                    5  t          ||           d d d            n# 1 swxY w Y   |j        j        |j                 j        }|                    di           }||d<    |d         |                    dg           i |}t          ||           d S )Nr*   rc  r  r   r   rF   r0   r   r4  r   rt  rc  rs  r   r]   ri  funcr  )r   r   r  r   r   r   r   r   rM   r   rI  r   r  r  r  creation_inforW  )	r   r=   r   ri  rq  r   r]  r]   rt  s	            r"   test_layer_creation_infor  
  s   	E"IIUENQ,>??	@	@BN21%%%00vSE 1   
 ""G?6&'BBBDS	))++++++ ?6&111D	~	&	&  $              
 9DJ'5DXXh##FF94<&"--888DdDs   >CC"Cc                   t          j        dt          j        dt          j                  i          }t          j        |d          }|                    t          |           |           t          j
        d          5  t          j        t          |           |          }d d d            n# 1 swxY w Y   |                                j        }t          |          d	k    sJ t          t!          |                                                    }t%          |t&                    sJ |j        d
dik    sJ d S )Nr   r   rr   rI   r0   r   bar)r  rF   r  )r   r   r   r   r   r   r   r   r   r  annotater   r  r  rU   r  r  r  r  r   r   )r   r=   r   r  r   r  layers          r"   "test_blockwise_parquet_annotationsr  
  s[   	sBIb999:	;	;B^BA...F
c&kk&111	5	!	!	! : :oc&kk&999: : : : : : : : : : : : : : : !!(Fv;;!fmmoo&&''Ee-...........s   $B44B8;B8c                J   d}d}t          |           }t          j        dt          j        |t          j                  i          }t          j        ||          }|                    ||           t          j	        ||d          
                                j        }t          |          d	k    sJ t          t          |                                          d
         t                     sJ t#          |           d	z  |d	z  }dz  |dz  }
                                j        }t          |          dk    sJ t%          d |                                D                       sJ fdt'          |          D             }t)          
                                |          }	|	j        }t          |                                          d
         }
t          |          d	k    sJ t          ||
         t                     sJ t#          |           d S )Nr   rI   r   rr   r0   r   Tr  rF   r   r*   r  c              3  @   K   | ]}t          |t                    V  d S r8   )r  r   )r    r  s     r"   r   z2test_optimize_blockwise_parquet.<locals>.<genexpr>
  s,      IIz%++IIIIIIr$   c                "    g | ]}j         |fS r   )r  )r    r!   r   s     r"   r#   z3test_optimize_blockwise_parquet.<locals>.<listcomp>
  s    777qSYN777r$   )r   r   r   r   r   r   r   r   r   r   r  r  rU   r  r   r  r   r   r   r  r	   r  )r   r=   ry   r1   r   r   r  r  r  graphr-   r   s              @r"   test_optimize_blockwise_parquetr  
  s   DK
f++C	sBId"(;;;<	=	=B^BK888F
c&)))
/#f$
G
G
GC !!(Fv;;!d6==??++A.	::::: c6 1HC
aKF 2IC
bLF !!(Fv;;!IIIIIIIIII 8777E+$6$6777Ds1133T::E\Fq!Dv;;!fTlI..... c6r$   c                   t          |           }t          j        t          j        dt          j                  t          j        dt          j                  d          }d|j        _        t          |          dz  }t          j        |j        d|         d                              |dd	
           t          j        ||d          }|j        dk    sJ t          j        ||dd          }|j        dk    sJ t          j        |j        |d         d                              |ddd           t          j        ||dd          }|j        dk    sJ t          j        ||dd          }|j        dk    sJ dS )z(Test split_row_groups read_parquet kwarg   rr   r  r/   rI   Nr0   r5   r_  r  Tr=   r  r   F)r=   r   r  r  rZ   r=   r  rU  )r   r   r   r   r   r   r   r/   r-   rU   r   r   rn  r   r   r1   )r   r=   r   r   rp  rt  s         r"   test_split_row_groupsr  
  s    f++C		#RX...RYs"*5U5U5UVV
 
B BHMr77a<DN275D5>q111<<Ic =    ?3vEEEDq    ?Fu  D q    N27455>q111<<D2 =    ? 	  D r!!!!?Fu  D q      r$   r  rU  r   c                   t          |           }d}d}d}t          j        t          j        d|z  t          j                  t          j        d|z  t          j                  d          }t          |          dz  }	t          j	        |j
        d |	         |                              |d|	           t          j	        |j
        |	d          |                              |d
d|           t          j        ||||          }
t          ||z            }|
j        dt          j        ||z            z  k    sJ d S )Nr*   r   r<  rI   rr   r  r0   r5   r  Tr  )r=   r  r   )r   r   r   r   r   r   r   rU   r   r   rn  r   r   r  r1   mathceil)r   r  r   r=   r   r  r1   	half_sizer   rp  r   expected_rg_couts               r"   test_split_row_groups_intr  
  sc    f++CNKI	9Q]"(;;;1y=
;;;	
 	

 
B r77a<DN275D5>{;;;FFIn G    N27455>{;;;FFD> G    ?)/	  D 9~566q49-=@P-P#Q#QQQQQQQQr$   rw  r  c                   d}d}t          j        t          j        |t          j                  t          j        |t          j                  d          }t          j        |d                              t          |           d|d	           t          j
        t          |           ||d
          }t          j        ||z  |z            }|j        |k    sJ t          |          |k    sJ t          ||d           d S )Nr*   r  rr   r  r   r0   r5   F)r=   r  r   T)r=   r  aggregate_filesr  )r   r   r   r   r   r   r   r   r   r   r   r  r  r1   rU   r   )r   r=   r  r  ry   r   r   npartitions_expecteds           r"   )test_split_row_groups_int_aggregate_filesr    s   
 ND	9T2224rz222	
 	

 
B N21%%%00FInRW 1    ?F)	  D  9d^&;?O%OPP33333t99b$E******r$   zfilters,op,length)r(  r{  r   c                &    | | d         dk             S Nr(  r   r   r  s    r"   r  r  D  s    a##& r$      )r(  rt  r   c                &    | | d         dk             S r  r   r  s    r"   r  r  H  s    q33'7 r$   rI   c                L   |dk    r>t          t          j                  t          d          k     rt          j        d           |                     d          }t          j        dd gdz  d gdz  z   t          j	        d          
                                d gz   dd gd	z  d gd
z  z   d          }|                    |dd           t          j        ||||          }t           ||                    |k    sJ t           ||           ||          d           d S )Nr5   8.0.00pyarrow>=8.0.0 needed for correct null filteringr   rF   r   r?  r   rI      r^  r*   r  r=   ri  r  Fr  )r  r  __version__rM   rH   r   r   r   r   r   rd  r   r   r   rU   r   )	r   ri  oplengthr  r=   r   r   rZ  s	            r"   test_filter_nullsr  >  s:    }R^<<}W?U?UUUFGGG;;~&&D	TQ$!+2%%''4&0tqD6B;.	
 	

 
B MM$yM<<<_)	  F rr&zz??f$$$$bbjj""R&&e444444r$   c                   t          t          j                  t          d          k     rt          j        d           |                     d          }t          j        ddd gdz  d gdz  z   i                              |dd	           t          j
        |ddd
t          j        fg|          }t          |          dk    sJ t          |d                                                                                   sJ t          j
        |dddt          j        fg|          }|d                                                                         dgdz  k    sJ d S )Nr  r  r   r   rF   r   r5   r*   r  isr  is not)r  r  r  rM   rH   r   r   r   r   r   r   r   r  rU   r   r   isnard  )r   r  r   result_isnaresult_notnas        r"   test_filter_isnar  d  so    R^$$}W'='===FGGG;;~&&DL#4y1}vz1233>>Yr ?    /tRV$%)	  K {r!!!!{3''))..0011111?x())	  L $$&&--//A37::::::r$   c                   t          |           }t          j        t          j        dt          j                  t          j        dt          j                  d          }d|j        _        d}dd|fg}t          j
        |d	                              |d
dd           t          j        ||          }t          j        ||d
d
|          }|d         |k                                                                    sJ t          ||d         |k                                             ||d         |k                                                        d S )Nr  rr   r  r/   r=  r~   rt  r   r0   Tr5   r  r  r   )r=   r   r  ri  r|   )r   r   r   r   r   r   r   r/   r-   r   r   r   r   anyr   r   )r   r=   r   r   
search_valri  r   rt  s           r"   test_split_row_groups_filterr    s_   
f++C		#RX...RYs"*5U5U5UVV
 
B BHMJT:&'GN21%%%00D2 1    ?3v...D?   D K:%**,,4466666T%[J&'//11T%[J&'//11    r$   c                l   t           j                            | d          }t          j        g dg dg dg ddg d          }|                    ||	           t          j        ||	          }|d
dg                             d          	                                
                                 d S )Npath.parquetr  r   rI   rF   rI   r   r   r   rI   r  r*  r^  r.   r   r   r   r  )r   r   r   r   r   r   r   r   rollingmaxr   )r   r=   r   r   r   s        r"   &test_optimize_getitem_and_nonblockwiser    s    7<<//D	iiiiiiiiiiiHHoo
 
 
B MM$vM&&&
/$v
.
.
.Cc
OA""$$,,.....r$   c                x   t           j                            | d          }t          j        g dg dg dg ddg d          }|                    ||	           t          j        ||	          }|d
                             |d                   	                                
                                }|d                             |d                   	                                
                                }|d
dg                             d                                          
                                }|                    d                                          
                                }t          j        ||z   |z   |z             \  }	t          j        |          d         d         t          j        |          d         d         t          j        |          d         d         t          j        |          d         d         g}
t          |	|
          D ]\  }}t!          ||           d S )Nr  r  r  r  r  r*  r^  r.   r   r   r(  r   rI   r   )r   r   r   r   r   r   r   r   groupbyfirstrF  r  r  r  r   zipr   )r   r=   r   r   r   df2adf2bdf2cdf2drZ  r[  r   r   s                r"   test_optimize_and_notr    s   7<<//D	iiiiiiiiiiiHHoo
 
 
B MM$vM&&&
/$v
.
.
.Cs8CH%%++--88::Ds8CH%%++--88::DSz?""1%%))++6688D;;q>>**,,DTD[4/$677IV 	T1a T1a T1a T1a 	H FH%%  1!Q r$   c                .   t          j        t          j        d          t          j        d          d          }t          j        |d          }|                    | |d           t          j        | |d	
          }t          ||d           d S )Nr  rr   r  r   rF   r0   Trk  adaptiver  Fr  )r   r   r+  r   r   r   r   r   )r   rJ   rK   r   rq  r   s         r"   test_split_adaptive_emptyr     s    	BIE2229Q9Q9QRR	S	SB>"!,,,DOOF<TOJJJ?#  D
 dDe,,,,,,r$   r  r   	blocksize   1MiBc           	     N   |r|dk    r|st          j        d           d}t          j        t          j                            g d|          t          j                            |          t          j                            dd|          d          }t          j	        |d	
          }|
                    t          |           |||d           |r|nd}	t          |	t                    rZt          j        t          d          5  t          j        t          |           ||d|	          }
d d d            n# 1 swxY w Y   n&t          j        t          |           ||d|	          }
|dk    r|
j        |j        k     sJ n#|dk    r|r|
j        dk    sJ n|
j        dk    sJ |rd|
                                                    ddg          }|                    ddg          }t'          |ddg         |ddg         d           d S t'          ||
dd           d S )Nr2   z4Fastparquet requires _metadata for partitioned data.r_  applebananacarrotrx   rF   r   r^  ry  r0   F)r=   r  r   r   TBehavior may changerB  r  r=   r  r  r  r  r  r  r   r(  r  rQ  )rM   rH   r   r   r   r   r   rX  r   r   r   r   r  rJ  rK  r   r1   r   r  r   )r   r  r  rJ   rK   r  df_sizer  rq  r  r   r   s               r"   test_split_adaptive_filesr    s     L}44X4JKKKG
,!!"?"?"?g!NN!!w!//""1ag"66	
 	
 C >#1---DOOF!$     '3<llO/3'' 
\-/DEEE 	 	?F"#!+ /  D	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 KK'+
 
 
 D$"222222	f		 	)#q(((((#q((((  Hllnn((#s44oosCj))#sCj/3Sz?FFFFFF$eGGGGGGs   'D>>EEr  r   c           
        d}ddg}d}t          j        t          j                            g d|          t          j                            ddg|          t          j                            |          t          j                            d	d|          d
          }t          j        |d          }|                    t          |           ||d           t          j        t          d          5  t          j        t          |           ||d|          }	d d d            n# 1 swxY w Y   |dk    r|	j        dk    sJ n|dk    r|	j        dk    sJ |	                                                    ddg          }
|                    ddg          }t#          |ddg         |
ddg         d           d S )Nr  r   r   r_  r  rx   smalllargerF   r*  ry  r0   Fr=   r  r   r	  rB  r  r
  r  r   r(  r  r  )r   r   r   r   r   rX  r   r   r   r   rM   rJ  rK  r   r1   r   r  r   )r   rJ   rK   r  r  r  r  r  rq  r   r   s              r"   #test_split_adaptive_aggregate_filesr    s   
 I:LG
,!!"?"?"?g!NN!!7G"47!CC!!w!//""1c"88		
 	
 C >#1---DOOF!	     
m+@	A	A	A 
 
KK'+
 
 

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 #1$$$$$	C		1$$$$ ,,..
$
$c3Z
0
0C
//3*
%
%Cc3*osC:EBBBBBBs   9'D,,D03D0)N   r  r  c           	     P   d}d}d}t          j        t          j                            g d|          t          j                            |          t          j                            dd|          t          j        d|          d                              d	          }t          j	        ||
          }|
                    t          |           d||           |rt          |           }	nIt          |           }
t          j        |
          }d|vsJ t          j                            |
d          }	t          j        |	||ddd	d          }t#          ||d           ||z  }|s|j        |j        k    sJ d S |j        |k     sJ |dk    r|j        dk    sJ d S d S )NrI   r_  r   r  rx   rF   r   r   r   r(  r/   r/   r0   r5   r=   r  r   r   r   r  T)r=   r  r  r   r/   r  Fr   r  )r   r   r   r   r   rX  r   r   r   r   r   r   r   r   r   r   r   r   r1   )r   r  r=   r  npartsr  r  r   rq  r   dirnamer   r   r  s                 r"   test_split_adaptive_blocksizer  @  s    FGN	!!"?"?"?g!NN!!w!//""1ag"66Yq'**		
 	

 
 i  >"&111DOOF%$	      26{{f++
7##%''''w||G[11?#   D dD%0000.N )4#3333333 .0000 #q((((  )(r$   )r     r  r  c           	     0   d}d}d}t          j        t          j                            g d|          t          j                            |          t          j                            dd|          t          j        d|          d                              d	          }t          j	        ||
          }|
                    t          |           d||           |rt          |           }	nIt          |           }
t          j        |
          }d|vsJ t          j                            |
d          }	t          j        |	||d	          }t#          ||           |dv r|j        |j        k    sJ t          j                            t          |           d          }|
                    ||           t'          |j                  D ]}t          j                            |d| d          }|dk    r3t)          j        |          }t-          d |j        D                       }nFt1          j        |          j        t-          fdt'          j                  D                       }||k    sJ d S |j        |j        k    sJ d S )NrI   r  rF   r  rx   r   r   r  r/   r0   r5   r  r   r   )r=   r  r/   )r  r  r   r   r6  r7  r2   c                    g | ]	}|j         
S r   )total_byte_size)r    rgs     r"   r#   z"test_blocksize.<locals>.<listcomp>  s    IIIRb0IIIr$   c                D    g | ]}                     |          j        S r   )r  r  )r    r  r  s     r"   r#   z"test_blocksize.<locals>.<listcomp>  s8        R((8  r$   )r   r   r   r   r   rX  r   r   r   r   r   r   r   r   r   r   r   r   r1   r  r2   r  sumr  r  r  r  )r   r  r=   r  r  r  r  r   rq  r   r  r   r   outpathr!   r   r  sizep0r  s                     @r"   test_blocksizer"  y  s    FGN	!!"?"?"?g!NN!!w!//""1ag"66Yq'**		
 	

 
 i  >"&111DOOF%$	      26{{f++
7##%''''w||G[11 ?	  D b$K$"22222',,s6{{E22///t'(( 	' 	'Ag':q':':':;;B&& ,R00II2=IIIJJ^B''0   "'(9":":    Y&&&&&	' 	'  4#3333333r$   c                D   t          |                     d                    }t                                          }d|j        _        |                    ||                    d          rdnd           t          j	        ||dddd          }t          ||           d S )	Nr   r/   r5   r2   r   z10 kiBT)r=   r  r   r  r/   r  )r   rJ   rK   r   rZ  r  s         r"   test_roundtrip_pandas_blocksizer$    s    v{{>**++D
''))CCINNN,"9"9)"D"DWYY-        H c8r$   c                   t          j        d t          d          D                       }t          |           }|                    |dgd           t          j        ||          }t          |d                                                   dk    sJ t          |                                j	                  dk    sJ d S )Nc                F    g | ]}t          |          |d |dz           dS )ABCr  )r   r  group)r   r   s     r"   r#   z<test_read_pandas_fastparquet_partitioned.<locals>.<listcomp>  s0    LLLaQE!a%L	9	9LLLr$   r   r(  r2   )partition_colsr=   r   )
r   r   r  r   r   r   r   rU   r   r(  )r   r=   rZ  r   r  s        r"   (test_read_pandas_fastparquet_partitionedr*    s    
,LL588LLL C v;;DNN4	-NHHHtF333Hx ((**++q0000x!!'((A------r$   c                p   t          j        g dg dd          }t          j                            t          |           d          }|                    d          rdnd}|                    ||           t          j	        ||          }t          j        |d         |          \  }}|d         }t          |j
        |j        d	fg
          }d |j        D             d	         }	|j        |	         }
t          |
t                     sJ |dk    rt#                      rddgndg}|
j        |k    sJ d S )Nrd  r  r  data.parquetr5   r2   r   r  r   r  c                <    g | ]}|                     d           |S )r  )r  )r    rb   s     r"   r#   zLtest_read_parquet_getitem_skip_when_getting_read_parquet.<locals>.<listcomp>  s)    HHHC)G)GHCHHHr$   r  )r   r   r   r   r   r   r  r   r   r   r  optimizer   r  r  r  r   r   r   )r   r=   rZ  r   	pd_enginer   r   r   r  r  r  r[  s               r"   8test_read_parquet_getitem_skip_when_getting_read_parquetr0    sL   
,///6T6T6TUU
V
VC7<<F^44D#..y99L		}INN4	N***
/$v
.
.
.C=S3''DAq c(C
$SXci^4D
E
E
ECHH3:HHHKDz$Hh 011111--2I2K2K-c

RUQV  x''''''r$   c                $   t          |           } t          j        g ddz  t          j        d          t          j                            ddgd          d          }t          j        |d          }|	                    | d	d
g|           t          j
        | d	||dg          }|                                }t          |          t          ||d         dk                        k    sJ |d                                         dk     sJ d S )Nr  r      rc  r  rx   )r  timer   r0   Fr  r  )r3  ru  r   )r/   r=   r   ri  r3  r   )r   r   r   r   r   r   r   r   r   r   r   r   rU   r  )r   rJ   rK   r   df_write	ddf_writer  df_reads           r"    test_filter_nonpartition_columnsr7    s,   
 [[F|,,"IbMMi&&u~B&??	
 	
 H xQ777IE|     /"$  H   Gw<<3wwv':;<<<<<<6?  1$$$$$$r$   c           	     R   t          |           } t          j        t          j        t          j        g dd          t          j        g dd          d          d          }|                    | d	
           t          j        | d	d          }t          ||d           d S )N)rF   NrI   r!  rr   )r  rc  Nr   r  rF   r0   r5   r   Tr  Fr  )	r   r   r   r   r   r$  r   r   r   )r   rq  r   s      r"   %test_pandas_metadata_nullable_pyarrowr9    s    [[F>
Xlll':::X222%@@@ 	
 	
   D 	OOF9O---?6)NNNDdDe,,,,,,r$   c                Z   t          j        t          j        d                    }t          rt	          j        t          d          }nt          j                    }|5  t          j	        |j
        dz   |j        dd          }d d d            n# 1 swxY w Y   |                    d          }t          j                            t          j        |          gdg	          }t          j                            ||  d
d           t&          t)          d          k     rt	          j        t          j        j                  5 }t1          j        t5          |           d                                           dt5          |j                  v sJ 	 d d d            n# 1 swxY w Y   n5t1          j        t5          |           d                                           ddlm}  G d d|          }t1          j        t5          |           |                                           d S )Nr   z!invalid value encountered in castrB  rI   r  )rk  stopnumrs   zdatetime64[ms]tsr   z/file.parquetF)use_deprecated_int96_timestampsrc  r5   r   zout of boundsr   re   c                  N     e Zd Zed	d            Ze	 	 d
d fd            Z xZS )Mtest_pandas_timestamp_overflow_pyarrow.<locals>.ArrowEngineWithTimestampClamparrow_tablepa.Tablereturnc                   g }|j         D ]C}t          j                            |j                  r|j        j        dv rdddd|j        j                 }|j        }|                    t          j                                                              }t          j
        t          j        d                    }|                    |j        |z  dz   |j        |z  d           t          j        |t          j                              }|                    |          }|                    |           .|                    |           Et          j                            ||j        	          S )
zConstrain datetimes to be valid for pandas

            Since pandas works in ns precision and arrow / parquet defaults to ms
            precision we need to clamp our datetimes to something reasonable)r  msusl    d(	 rb  rq   r   rF   T)r  upperr  r   )r   r  typesis_timestampr   r  castr   r  r   iinfors   clipminr  r$  rZ   r  r   r  )	clsrA  new_columnsr  
multiplieroriginal_typeseriesr]  	new_arrays	            r"   clamp_arrow_datetimeszctest_pandas_timestamp_overflow_pyarrow.<locals>.ArrowEngineWithTimestampClamp.clamp_arrow_datetimesC  sR    K"* , ,8((22 ,HM%666'5Ye!T!T"J %(HM(+(<(<(F(F(H(HF8BHW$5$566DKK"h*4q8"h*4 $     
 !# < <I )} = =I&&y1111&&s++++8'';;S'TTTr$   NFpd.DataFramec                j    |                      |          } t                      j        ||f||d|S )N)r.  convert_string)rT  super_arrow_table_to_pandas)rN  rA  rb  r.  rW  r]   fixed_arrow_table	__class__s          r"   rY  zdtest_pandas_timestamp_overflow_pyarrow.<locals>.ArrowEngineWithTimestampClamp._arrow_table_to_pandase  sV     !$ 9 9+ F F15771! ,-	 
   r$   )rA  rB  rC  rB  )NF)rA  rB  rC  rU  )__name__
__module____qualname__classmethodrT  rY  __classcell__)r[  s   @r"   ArrowEngineWithTimestampClampr@  B  sy        		U 	U 	U 
	UB 

  	 	 	 	 	 	 
	 	 	 	 	r$   ra  )r   rK  rs   r   rM   rJ  RuntimeWarningr  r  linspacerM  r  r   r  r  r   r$  r  r  rX  r  r   libArrowInvalidr   r   r   r   r}  rh   rf   )	r   r]  r  arr_numeric	arr_datesr;  r|  ArrowEnginera  s	            r"   &test_pandas_timestamp_overflow_pyarrowri  $  s   8BHW%%&&D  'l>1TUUU$&&	 
 
k(Q,TX4w
 
 

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 ""#344IH  "(9"5"5!6tf EEEJ&'''     |4444]26.// 	31OCKK	:::BBDDD"c!'ll22222	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	FI666>>@@@QQQQQQ3 3 3 3 3 3 3 3l OCKK(EFFFNNPPPPPs%   &BBBAF!!F%(F%c                   t          j        dt          j        d          gi          }t          |                     d                    }|                    ||           ddi} t          j        |                                          j	        d	i |}t          j        |d|          }t          ||           |j        j        |                                j        j        k    sJ d S )
Nr  rh  r   r   timestamp_as_objectTr5   )r=   rT  r   )r   r   r  r   r   r   r  r  r  r  r   r   r   r  rs   r   )r   r=   r   r   rT  r  gots          r"   test_arrow_to_pandasrm  {  s    
 
sR\,7789	:	:Bv{{>**++DMM$vM&&&,d3O2R^D!!&&((2EE_EEF
/$y/
R
R
RCfc5;#++--////////r$   
write_colsr  r  r  r`   r  c                   |                      d           |                      d           t          |                      d                    }t          |                      d                    }t          j                            |d          }t          j                            |d          }t          j        ddt          d          d	          }t          j        d
dt          d          d	          }||         }||         }|                    |d           |                    |d           |dk    r||g}	nt          |           }	t          j	        ||gd          }
|dk    rt          t          d          k    rjt          j        |	|          }|                                                    d                                          |
                                k    sJ d S |g d	k    r*t          j        |	|          }t#          ||
d           d S t%          j        t(                    5  t          j        |	|           d d d            d S # 1 swxY w Y   d S )Npart=apart=bzpart=a/kind=xzpart=b/kind=xr,  r   r'   r   ro  r   Fr.   r2   Tignore_indexz0.8.3r   r   r  )r  r   r   r   r   r   r   r  r   r  r  r  r   r   r   r   to_dictr   rM   r   rW   )r   r=   rn  path0r5  _df1_df2r  r   r   r  rZ  s               r"   test_partitioned_column_overlapry    s   
 LL
LL_--..E_--..EGLL//EGLL//E<c%((CCDDD<c%((CCDDD
z
C
z
CNN5N&&&NN5N&&&u~6{{Yd|$777F#6w9O9O#O#Of555~~+++66>>@@FNNDTDTTTTTTT	...	.	.f555&&e444444 ]:&& 	1 	1OD0000	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1s   <I  I$'I$c                   |                      d          }|                      d          }t          j                            |d          }t          j                            |d          }t	          j        dt          d          d          }t	          j        dt          d          d          }t          j        	                    ||         d	          
                    i 
          }t          j        ||           t          j        	                    ||         d	          
                    i 
          }t          j        ||           t	          j        ||gd          }	t          j        t!          |           |          }
|
d                             d          |
d<   t%          |
t'          |	j                           |	d           d S )Nrq  rr  r,  r   r   r  r  r   Fr   )r  Trs  r   r  r  r  )r  r   r   r   r   r   r  r  r  r   r9  r  r  r  r   r   r   r   r   r   r   )r   r=   rn  r5  r6  rw  rx  t1t2r  rZ  s              r"   #test_partitioned_no_pandas_metadatar~    s    LL""ELL""EGLL//EGLL//E
 <U1XX6677D<U1XX6677D			Z 
 
 
 r**  N2u			Z 
 
 
 r**  N2u Yd|$777F_S[[888FF^**844F6NfT&.))*FFFFFFFr$   c                   |                      d          }|                      d          }t          j                            |d          }t          j                            |d          }t	          j        dt          d          d          }t	          j        dt          d          d          }|                    |d           |                    |d           t	          j        ||gd	          }t          j
        t          |           dd
gt          |           d          }|d
                             d          |d
<   t          |t          |j                           |d           d S )Nr   r   r,  r   r{  r5   r   Trs  r  )partitioningpartition_base_dir)r=   datasetr  Fr  )r  r   r   r   r   r   r  r   r  r   r   r   r   r   r   r   )r   r5  r6  rw  rx  r  rZ  s          r"   #test_pyarrow_directory_partitioningr    sL    LLELLEGLL//EGLL//E<U1XX6677D<U1XX6677DOOE)O,,,OOE)O,,, Yd|$777F_F"(VMM  F
 F^**844F6NfT&.))*FFFFFFFr$   c                .   t          |           }d}d}t          j        |d                              ||z            }t	          j        t          j        |          t          j                            |          t	          j        |          d                              d          }d |j	        _
        t          j        ||          }|                    |d	|
           ||d	         dk             }	t          rm|dk    rg|	                                }	|	d	                             t	          j        |	d	         j        j                            d                              |	d	<   t          j        ||dg          }
t+          |	|
           d S )Nrq   r   r   rr   rx   )r+   r  r  r+   r0   r  r  rF   r2   r   )r  rt  rF   rs  )r   r   r   rW  r   r   r   rj  r   r/   r-   r   r   r   r   r  r   CategoricalDtypers   rb  r   r   )r   rJ   rK   r   ry   r1   r   r   r  r  rl  s              r"   test_partitioned_preserve_indexr    su   
f++CDK
	+W---44T[5HIIA<y!!t!,,""	
 	
  i	 	 DJO
.;
7
7
7CNN3SN>>>$s)q.!F 
55Sk((s 1 < C CG L LMM
 
s /#kN;K
L
L
LCfcr$   c                   |                     d          rt          j        dd           t          |                     d                    }t          j        ddgddgd	d
gd                              d          }d |j        _	        |
                    ||                     d          rdndd           t          j        |          }t          j        ||          }t          ||           d S )Nr5   0.15.0r  r   rF   rI   r   r   r   r   r^  r(  r2   Tr   r   )r  rM   r  r   r   r   r   r   r/   r-   r   r   r   r   )r   r=   r   r   r  rl  s         r"   $test_from_pandas_preserve_none_indexr    s    ## <I(;;;;	V[[((	)	)B	QF!Qq!f==	>	>	H	H	M	MBBHMMM
"--i88Kyym     _R  F
/"V
,
,
,Cfcr$   c                   |                     d          rt          j        dd           d}n
|dk    sJ d}t                              d          }t          |j                  D ]T\  }}|                     d| d          }|                                	                    t          |          |	           Ut          j        t          |           d
|          }t          ||           d S )Nr5   r  r  r2   Tr   ztest.r7  r   Fr   )r  rM   r  r   r   rG  rN  r   r   r   r   r   r   r   )r   r=   rJ   rq  r!   r  r   r   s           r"   %test_multi_partition_none_index_falser    s    ## %I(;;;; &&&&$ ???%%DT_-- B B4{{.1...//!!#d))L!AAAA ?3v;;eFCCCDdDr$   c                r   t          |                     d                    }t          j        dg dit          j        dd                    }|                    ||                    d          rdnd	
           t          j        ||
          }t          ||
                                           d S )Nr   tr  rF   r   )rk  r;  r.   r5   r2   r   )r   r   r   r   
RangeIndexr   r  r   r   r   r   )r   rJ   rK   r   r  r  s         r"   )test_from_pandas_preserve_none_rangeindexr  0  s     
V[[((	)	)B
,YYY'r}11/M/M/M
N
N
NCNN
 7 7	 B BU99     /"[
1
1
1Cc3;;==!!!!!r$   c                F   d}t          |                     d                    }t          j        dddg|ddgi                              d          }d |j        _        t          j        |d          }t          j
        t          |	          5  |                    ||d
           d d d            n# 1 swxY w Y   t          j        t                    5 }|                    ||           d d d            n# 1 swxY w Y   |t          |j                  v sJ d S )N__null_dask_index__r   r'   rF   rI   r   r   r0   rB  Fr   r   )r   r   r   r   r   r/   r-   r   r   rM   rJ  UserWarningr   r   rW   r}  )r   r=   	null_namer   r   r   r|  s          r"   test_illegal_column_namer  =  s    &I	V[[((	)	)B	sQFI1v6	7	7	A	A#	F	FBBHM
.
+
+
+C 
k	3	3	3 = =r&e<<<= = = = = = = = = = = = = = =
 
z	"	" *ar&)))* * * * * * * * * * * * * * *AG$$$$$$s$   B66B:=B:C>>DDc                   t          j        g dg dd          }t          j        |d          }|                    t          |           |d           t          j        t          |           |d	          }|j        d
k    sJ d S )N)rF   rI   NNr  r   rI   r0   Fr   r   r   )NNN)r   r   r   r   r   r   r   r   )r   r=   r   r   r  s        r"   "test_divisions_with_null_partitionr  R  s    	...\\\BB	C	CB
.
+
+
+CNN3v;;v5NAAAs6{{6EEEH!3333333r$   c                X   t          |           }t          j        g dg dd          }|                    dd          }t	          j        |d          }|                    ||	           t	          j        |d
d          }|                                 t          ||           d S )Nr   r   r   r   Tr   rI   r0   r   r5   r  )
r   r   r   r   r   r   r   r   r   r   )r   r=   r   r   r   r   s         r"   test_pyarrow_dataset_simpler  [  s    	VB	IIIOOO<<	=	=B	c	%	%B
.
+
+
+CNN2fN%%%obMMMGOOc7r$   test_filterc                   t          |           }t          j        g dg dd          }|d                             d          |d<   t	          j        |d          }|                    ||dd	           t	          j        |d
|rdgnd d          }|rFt          ||d         dk             	                                |	                                           d S t          ||           d S )Nr   r   r   r   r`  rI   r0   T)r=   r  r   r5   r   rt  r   r  r   
r   r   r   r   r   r   r   r   r   r   )r   r=   r  r   r   r   r   s          r"    test_pyarrow_dataset_partitionedr  g  s     
VB	IIIOOO<<	=	=BgnnZ((BsG
.
+
+
+CNN2f3DNQQQo
&1;!""t 	  G   #c#h#o&..00'//2C2CDDDDD#wr$   c                   d}t          j        t          j        g dd          t          j        g dd          d          }t          j        |d          }|                    t          |           |d	
           t          j        t          j	        
                    | d                    }t          |          dk    sJ t          j        t          |           |dddt          j        d	t          j                    fg          di          }t           j        dk    r|j                            d          |_        t)          |dd	g         |dd	g         d|           d S )Nr5   )r   rF   Nr!  rr   r  )r  r'   rF   r0   r  r4  zid=*/*.parquetr  rQ  r  hiveflavorr  )r=   r.  r  rU  r'   F)r   r   )r   r   r+  r   r   r   r   r  r   r   r   rU   r   r  r  r   rX  rY  r/   r   r   )r   r   r=   r   r   r  r  s          r"   test_null_partition_pyarrowr  |  sv    F	)LLL888999G444	
 	

 
B .
+
+
+CNN3v;;vDNAAA
)BGLL)9::
;
;Cs88q==== F& )dBHJJ%7$899 
	
 
 
H "" I$$W--	S$K#t	     r$   c                ^   t          |           }t          j        g dg dd          }|d                             d          |d<   t	          j        |d          }|                    |dd	           t          j        t                    5  t	          j
        |dd
gd          }d d d            n# 1 swxY w Y   t	          j
        |dd
g          }t          ||           t          ||d         dk                                             |                                           d S )Nr   r   r   r   r`  rI   r0   r5   r4  r  F)r=   ri  read_from_pathsrs  r   )r   r   r   r   r   r   r   rM   rJ  rK  r   r   r   )r   r   r   r   	read_df_1	read_df_2s         r"   $test_pyarrow_dataset_read_from_pathsr    sl   	VB	IIIOOO<<	=	=BgnnZ((BsG
.
+
+
+CNN2icN:::	m	$	$ 
 
O%&!	
 
 
	
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
!"  I i###c#c(c/"**,,i.?.?.A.ABBBBBs   B66B:=B:c                   t          |           }t          j        g dg dg dd          }|d                             d          |d<   t	          j        |d          }|                    |d	dd
g           t	          j        |d	|dg          }t          |	                                dg         ||d         dk             dg         d           d S )Nr   r   )r  r  r  r^  r   r`  rI   r0   r5   r(  r4  )r   rt  r   )r=   r  ri  r   r   Fr  r  )r   r  r   r   r   r   s         r"   'test_pyarrow_dataset_filter_partitionedr    s    
VB	  	
 	

 
B gnnZ((BsG
.
+
+
+CNN2isCjNAAA o
) 	  G 3% 
2c7a<#     r$   c                D   t          j        t          d          t          d          d          t	          j        fdt          d                    }|                    | |dg           t	          j        | |dg          }|d                             d	          |d<   t          j
        d
d         |           |dk    rVt	          j        | |dgg          }|d                             d	          |d<   t          j
        d
d         |           d S d S )Nr   abcdefg)r_   r  c                (    j         | | dz            S )NrF   )rn  )r!   r   s    r"   r  z<test_pyarrow_dataset_filter_on_partitioned.<locals>.<lambda>  s    "'!a!e)$ r$   r  r4  )r  rt  r(  rs  r  rI   r  r5   )r   r   r  r   r   from_mapr   r   r   r   rn  )r   r=   r   read_ddfr   s       @r"   *test_pyarrow_dataset_filter_on_partitionedr    s>   	eAhhY@@	A	AB
+$$$$a C NN6&xN@@@ $%  H
  '..x88HVbgaclH%%% ?)*+
 
 

 $F+228<<"'!A#,))))) r$   c                   t          |           }  t          j        t          j        j                  g g dd          } t          j        t          j        j                  g dg ddd          } t          j        t          j        j                  g dg ddd          }t          j        |||g          }|                    | dd	gd
d           t          j
        |           }d|v sJ d|v sJ t          j        t          j                            | d                    j                                        }|j        }|sJ |                    dd
          sJ d S )Nr&   )r  r  rr   rF   rF   rI   rI   rF   r   rF   r   r   r   rF   rI   rF   rI   r5   r'   FTr=   r  rZ   r   r   r   r  )r   r  rE  r   r   	from_dictr   rH  r   r   r   r  r  r   r   r  to_arrow_schemar  rW  )r   df_adf_bdf_cr   r   schema_commonr  s           r"   )test_parquet_pyarrow_write_empty_metadatar    s    [[F/4<.//r.  D 04<.//ll..6H  D 04<.//ll..6H  D 
$d+	,	,BMMU      JvE%&&&& N
V/00 __  $3O6666666r$   c                   t          |           }  t          j        t          j        j                  g dg ddd          } t          j        t          j        j                  g dg ddd          }t          j        ||g          }|                    | dd	gd
d            t          j        t          j        j                  g g dd          } t          j        t          j        j                  g dg ddd          }t          j        ||g          }|                    | dd	gddd           d S )Nr  r  r&   r  rr   r  )rI   r   rI   r   r5   r'   FTr  )r  r  r   r   )r=   r  rZ   r  r   )	r   r  rE  r   r   r  r   rH  r   )r   r  r  r  r  df_dr   s          r"   0test_parquet_pyarrow_write_empty_metadata_appendr     s    [[F/4<.//ll..6H  D 04<.//ll..6H  D /4,
'
'CNNU      04<.//r"4  D 04<.//ll..6H  D /4,
'
'CNNU       r$   c                D   t          |           } t          j        t          d          g ddz  d          }d|j        _        t          j        |d          }|                    | d||	           |r6t          j	        t          j                            | |d
z                       }n2t          j	        t          j                            | d                    }t          j        j                            |dd           t          j        | dd|d          }|r[|                    d          }|                                                    d          }|j                            d          |_        t+          ||           t          j        j                            |ddd          }t-          j        t          j                            | d                    j        }	|j        |	j        k    sJ |j        |	j        k    sJ |j        |	j        k    sJ d S )Nr_  r  r  r   r   r+   r*   r0   F)r   r  r=   z=*/*.parquetr   r5   r  )r=   split_everyT)r   r  r=   r/   r   r  )r=   r  out_dirr   )r   r   r   r  r/   r-   r   r   r   r  r   r   r   ior  create_metadata_filer   r  r   r   r   r   r  r  r  num_rowsnum_columnsr  )
r   rJ   rK   r  r  rq  r  r   r  fmd_files
             r"   test_create_metadata_filer  G  s    [[F ,U3ZZ.B.B.BR.GHH
I
ICCIN>#2...DOO!!	      ;iV\N-JKKLLiV[99::EM&& '    ?   D  )s##||~~))#..x((dD %-
,
,	 -  C ~bgll6;??@@IH<8,,,,,?h22222!8888888r$   c           	        t          j        t          j        t          j                            ddd          g d          d          }|                    d	
          }t          j        || |d	           |	                    d          }t          j        || |d	           t          j        |           }d |D             }t          |          |j        k    sJ d S )Nr   r_  )r_  r*   )lowhighry   )
r  r  rQ  r  rG  rH  GHIJr  r   r0   Tr   r=   	overwriter  c                    g | ]}|d v|	S ))r   r   r   r    r~   s     r"   r#   z5test_read_write_overwrite_is_true.<locals>.<listcomp>  s#    LLL1*K!K!KQ!K!K!Kr$   )r   r   r   r   r   r   rX  r   r   r  r   r   rU   r1   )r   r=   r   r   r   s        r"   !test_read_write_overwrite_is_truer    s     .
I!#I>>FFF	
 	
 	
   C //t/
$
$CM#vf==== ??q?))D M$v>>>> JvELLLLLEu::)))))))r$   c           
        ddl m} t          j        t	          j        t	          j        dd          t	          j        dd          t	          j        dd          f                    }g d|_        t          j	        |d	          }t          j
        || |d
dgd            ||                               d          }d |D             }|                    d	          }t          j
        || |d
dgd            ||                               d          }d |D             }	t          |	          t          |          k     sJ d S )Nr   )rs  )r  r  rF   )ry  r  rI   rP  r   r0   r  r  T)r=   r  r  rC  c                6    g | ]}|                                 S r   as_posixr  s     r"   r#   zBtest_read_write_partition_on_overwrite_is_true.<locals>.<listcomp>  s     ***aQZZ\\***r$   r  c                6    g | ]}|                                 S r   r  r  s     r"   r#   zBtest_read_write_partition_on_overwrite_is_true.<locals>.<listcomp>  s     ,,,qajjll,,,r$   )rr  rs  r   r   r   vstackfullr   r   r   r   rglobr  rU   )
r   r=   rs  r   r   files_r   r   files2_files2s
             r"   .test_read_write_partition_on_overwrite_is_truer    sa    

	######	
 	

 
B !BJ
.
+
+
+CM#vfC:QUVVVV T&\\$$F**6***E??q?))D
 M$vS#JRVWWWWd6ll  %%G,,G,,,F v;;U######r$   c                Z   t          j        dt          d          i          }t          j        |d          }t
          j                            t          |           d          }|	                    ||           t          j
        ||d                              d	
          }t
          j                            t          |           d          }|	                    ||d           |	                    ||d           t          |t          j
        ||d                     d S )Nr      rw  r0   r   r   r  r  1GB)partition_sizepath_newTr  F)r   r   r  r   r   r   r   r   r   r   r   r  r   )r   r=   r   r   r   r   r  s          r"   -test_to_parquet_overwrite_adaptive_round_tripr    s   	sE#JJ'	(	(B
.
+
+
+C7<<FV,,DNN4N'''?#   kk''	 	
 w||CKK44HOOHVtO<<<OOHVtO<<<
"	
 	
 	
    r$   c                   t          j        dt          d          i          }t          j        |d          }t          j        t                    5  t          j        |d|d           d d d            n# 1 swxY w Y   t          j        t                    5  t          j        || |dd           d d d            d S # 1 swxY w Y   d S )	Nr   rU  r  r0   z./Tr  )r=   rZ   r  )	r   r   r  r   r   rM   r   rW   r   r   r=   r   r   s       r"    test_to_parquet_overwrite_raisesr    sd    
sE"II&	'	'B
.
+
+
+C	z	"	" @ @
c4$????@ @ @ @ @ @ @ @ @ @ @ @ @ @ @	z	"	" O O
c6&NNNNO O O O O O O O O O O O O O O O O Os$   A88A<?A<CC
Cc                b   |                      d          }t          j        t          j        dt          d          i          d                              ||           t          j        |          }|| fD ]}t          j	        t          d          5  |                    |d	
           d d d            n# 1 swxY w Y   |                    |j        dz             }t          j	        t          d          5  |                    |d	
           d d d            n# 1 swxY w Y   d S )Nsubdirr'   ry  rI   r0   r   zsame parquet filerB  T)r  rF   )r(   )r  r   r   r   r   r  r   r   rM   r   rW   r  r'   )r   r=   r  r   targetr   s         r"   Etest_to_parquet_overwrite_files_from_read_parquet_in_same_call_raisesr    s    \\(##FN2<eBii 011qAAALLv M    /&
!
!C 6" 4 4]:-@AAA 	3 	3NN6TN222	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 zzCEAIz&&]:-@AAA 	4 	4OOFdO333	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	44 4s$   B;;B?	B?	?D##D'	*D'	c                j   t          j        t          d          t          d          d          }t          j        |d          }t          j        t          d          5  |                    t          | 
                    d                    |           d d d            d S # 1 swxY w Y   d S )	Nr*   )r'   rF   rI   r0   znon-string column namesrB  tempr   )r   r   r  r   r   rM   r   rW   r   r   r   r  s       r"   .test_to_parquet_errors_non_string_column_namesr    s    	E"II%))44	5	5B
.
+
+
+C	z)B	C	C	C @ @s6;;v..//???@ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @s   #8B((B,/B,c                   t           j                            ddddddt          j        dddd	d
ddt          j        dddddddddd          }t
          j                            |d          }|                    | d|           t          j
        | dg|          }|j                            d          |d<   t          |||j        dk                        d S )Ng     u@g     0v@g     `v@g     p@g     p@)r   rF   rI   r  r   r   r   g     @g     @g     T@g     @g     @g     @i    )r  r  yearrF   r0   r  r  )r  rt  r  )ri  r=   r   )r   r   r  r   r  r  	dataframer   r   r   r   r  r   r   r  s        r"   test_dir_filterr    s   			 6  6  $4DTdSS'	
 	

 
B. .
$
$RQ
$
7
7CNN6vN>>>?6,@+A&QQQD9##G,,DLdBrw$'(((((r$   c           	        t          |           } d t          d          D             }t          j        t	          j        |          d          }|                    | ddt          j        dd          i	           t          j	        | d
          }|d         j
        |d         j
        k    sJ t          ||d           d S )Nc                Z    g | ](}t          j        d d          t          d          d)S )r  zEurope/Berlinr  z123.00)r=  col1)r   rA  r   r   s     r"   r#   z0test_roundtrip_decimal_dtype.<locals>.<listcomp>&  sN       
  .?CCCH%%	
 	
  r$      rF   r0   r5   r  r   rI   )r   r=   r  r   Fr   )r   r  r   r   r   r   r   r  
decimal128r   rs   r   )r   r   rq  r   s       r"   test_roundtrip_decimal_dtyper  !  s     [[F 
 r  D >",t,,!<<<DOO	62=QRTUCVCV:WOXXX?6)444D<f!33333dD%000000r$   c                   t           j                            t          |           d          }t	          j        g dt          j                            d                    }|	                    |           t          j        ||          }g d|_        g d|_        t          ||                                           d S )Nr   r^  )r*   r  rx   )r   r   r   )r  r|  r~   )r   r   r   r   r   r   r   r   uniformr   r   r   r   r   r   )r   r=   r   r  r   s        r"   test_roundtrip_rename_columnsr  6  s     7<<F^44D
,RY5F5FG5F5T5T
U
U
UCNN4 ?4///D"??DL!//CKc4<<>>"""""r$   c                x   ddi}t          |           }t          j        t          d          t          d          d          }t	          j        |d                              |||d           t          |t	          j        ||	                     t          rt          j
        t          j                            |d
                    }|t          j                            |d          gz  }|D ]K}t          j        |          j        j        }|                                D ]}||         ||         k    sJ Lddi}t#          j        t&                    5 }	t	          j        |d                              |||           d d d            n# 1 swxY w Y   dt          |	j                  v sJ d S )Ns   my_keys   my_datar*   r   rI   r0   T)r=   custom_metadatar   r   r   r   s   pandass   my_new_pandas_md)r=   r  zUser-defined key/value)r   r   r   r  r   r   r   r   r   r  r  r   r   r   r  r  r  rM   r   rW   r}  )
r   r=   r  r   r   r   r   _mdrO   r|  s
             r"   test_custom_metadatar  E  s    !*-O v;;D	E"IIE"II66	7	7BN21%%%00' 	 1    b"/$v666777 
 4 	"',,t[99::"',,t[1122 	4 	4B.$$-6C$))++ 4 41v!3333334
 !"56O	z	"	" 
a
rq)))44+ 	5 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 $s17||333333s   -FFF)TFNc                H   t          |           } t          j                            | d          }t          j                            | d          }t	          j        t          d          ddgdz  d          }t          j        |d	          }|	                    ||d
           |	                    ||d
           dt          j
        |          vsJ t          t          j                            |d          d          5 }|                    d           d d d            n# 1 swxY w Y   dt          j
        |          v sJ dt          j
        |          vsJ t          j        ||d|          }t          j        ||d|          }	t          ||	           d S )Ndata1data2r_  r  rc  r  r   rI   r0   Fr   r=   r   r   r@   zINVALID METADATAT)r=   ignore_metadata_filer   )r   r   r   r   r   r   r  r   r   r   r   ro  r  r   r   )
r   r=   r   dataset_with_bad_metadatadataset_without_metadatar  rq  r~   ddf2addf2bs
             r"   test_ignore_metadata_filer  o  s   [[F "VW = =!w||FG<< ,U3ZZuenr.ABB
C
CC>#1---DOO&v5     	OO%f%    
 bj)BCCCCCC	bgll4kBBC	H	H $A	"###$ $ $ $ $ $ $ $ $ $ $ $ $ $ $"*%>??????bj)ABBBBBB O!!/	  E O !/	  E eUs   D''D+.D+r   r  c                x   t          |           } t          j        t          d          ddgdz  d          }t	          j        |d          }|                    t          |           ||           t	          j        t          |           |d	
          }t	          j        t          |           |d	|          }t          ||           t          j
                            d|i          5  t	          j        t          |           |d	
          }d d d            n# 1 swxY w Y   t          ||           d S )Nr_  r  rc  r  r   r*   r0   r  Tr  r  z*dataframe.parquet.metadata-task-size-local)r   r   r   r  r   r   r   r   r   r  rH  rV   )	r   r=   r   r  r  rq  r  r  ddf2cs	            r"   test_metadata_task_sizer    s    [[F
,U3ZZuenr.ABB
C
CC>#2...DOO[[=P    
 OF   E
 OF -	  E eU		57IJ
 
 
 
 KK $
 
 

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 eUs   .%DD#&D#)r   Nc           	        t          |           } t          j        t          d          ddgdz  d          }|                    |j                            d                    }t          j        |d	          }|	                    | |d
|           t          t          j                            | d          d                                           t          t          j                            | d          d                                           t          j        t          j                            | d                     t          j        | |d
          }t#          ||d           t#          |j        |j        d           dd}t          j        | fd|i |d          dd
i}t#          ||d           t#          |j        |j        d           t%          j        t(          d          5  t          j        | fd|i |dd
          dd
i}d d d            n# 1 swxY w Y   t%          j        t,          t.          j        j        f          5  t          j        | fd|i |d                                            d d d            n# 1 swxY w Y   t%          j        t6                    5  t          j        | fd|i |d                                           d d d            d S # 1 swxY w Y   d S )Nr_  r  rc  r  r   r`  )r   rI   r0   T)r=   r   r  _SUCCESSr@   zpart.0.parquet.crcr   r  F)check_categorical)check_category_orderc                    |rdd| iind| iS )Nr  require_extensionparquet_file_extensionr   )r_   legacys     r"   _parquet_file_extensionz0test_extra_file.<locals>._parquet_file_extension  s)    
 1Y,c233*C0	
r$   r=   r7  r   zrequire_extension is deprecatedrB  )r  .foo)F)r   r   r   r  r  r   r   r   r   r   ro  r   r   r   closeremover   r   rM   rJ  rK  r   OSErrorr  rd  re  r   rW   )r   r=   r  r   r   r   r  s          r"   test_extra_filer    s   
 [[F	E#JJeU^b-@AA	B	BB	RT[[,,	-	-B
.
+
+
+CNN !	     	fj	)	)3//55777f2	3	3S99??AAAIbgll6;//000
/&T
J
J
JCc2////ceRT6666
 
 
 
 /  "
!*
-
-  !	  C c2////ceRT6666 
m+L	M	M	M 
 
o
 

 &%j>>>
 
 !%	
 
 

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
!45	6	6  
	
 	
!	
%<%<T%B%B	
 	

')))               
z	"	"  
	
 	
!	
%<%<V%D%D	
 	

')))                 s6   ;$H++H/2H/ 0JJ #J  0K==LLc           	         t           j                            t          |           d          }t	          j        dt          d          i          }|                    ||           t          |t          j
        ||dd                     d S )Nz	multi.foor   r*   r   FTr   )r   r   r   r   r   r   r  r   r   r   r   )r   r=   r   r  s       r"   test_unsupported_extension_filer     s     
c&kk;	/	/B
,U2YY'
(
(CNN2fN%%%R_ReQUVVV    r$   c                   t          |           }t          j        t          j        dt          d          i          d          }|                    ||d d           t          |t          j        ||d                     d S )Nr   r*   rF   c                    d|  dS )Nr6  r  r   )r!   s    r"   r  z0test_unsupported_extension_dir.<locals>.<lambda>  s     r$   T)r=   name_functionr   r  )	r   r   r   r   r   r  r   r   r   )r   r=   r   ddf0s       r"   test_unsupported_extension_dirr%    s     v;;D>",U2YY'788!<<DOO// 	     dBODTRRRSSSSSr$   c                \   t          |           }t          j        g dg dd          }t          j        |d          }|                    |dd |           t          j        |          }d	|v sJ d
|v sJ d|v sJ d|v sJ t          |t          j	        ||d                     d S )Nr  r   rw  ry  r*   num1num2rI   r0   Tc                    d|  dS )Nhi-r7  r   r  s    r"   r  z&test_custom_filename.<locals>.<lambda>%  s     1a 1 1 1 r$   r   r#  r=   r   r   hi-0.parquetzhi-1.parquetr  )
r   r   r   r   r   r   r   r   r   r   )r   r=   r   rZ  r   r   s         r"   test_custom_filenamer/    s    	VB
,}}}55 C 
	+	+	+BMM
 11	     JrNNE&&&&%U""""U""""b"/"VNNNOOOOOr$   c                n   t          |           }t          j        g dg dd          }t          j        |d          }|                    |dd |           t          j        d	gd
gd          }t          j        |d          }|dk    rt          j        d           |                    |d |dd           t          j	        |          }d|v sJ d|v sJ d|v sJ d|v sJ d|v sJ t          j        g dg dd          }t          j
        ||d          }t          ||d           d S )Nr  r'  r(  rI   r0   Tc                    d| dz   dS Nr,  rI   r7  r   r  s    r"   r  zMtest_custom_filename_works_with_pyarrow_when_append_is_true.<locals>.<lambda>:       5a!e 5 5 5 r$   r-  !   ,   rF   r2   a  fastparquet errors our with IndexError when ``name_function`` is customized and append is set to True.  We didn't do a detailed investigation for expediency. See this comment for the conversation: https://github.com/dask/dask/pull/7682#issuecomment-845243623c                    d| dz   dS r2  r   r  s    r"   r  zMtest_custom_filename_works_with_pyarrow_when_append_is_true.<locals>.<lambda>J  r3  r$   )r#  r=   rZ   r  r   r   r.  zhi-2.parquetzhi-4.parquet)rF   rI   r  r   r4  )r   rw  ry  r*   r5  Fr   r  )r   r   r   r   r   r   rM   rG   r   r   r   r   )r   r=   r   rZ  r   r   expected_pdfactuals           r"   ;test_custom_filename_works_with_pyarrow_when_append_is_truer9  1  s   	VB
,}}}55 C 
	+	+	+BMM
 55	     ,t$$ C 
	+	+	+Bs	
 	
 	

 MM
55     JrNNE&&&&%U""""U""""U""""<!!!+<+<+<== L _Re<<<Ffl666666r$   c                   t          |           }t          j        g dg dd          }t          j        |d          }t          j        t          d          5  |                    |d|	           d d d            n# 1 swxY w Y   t          j        t          d
          5  |                    |d |	           d d d            d S # 1 swxY w Y   d S )Nr  r'  r(  rI   r0   z7``name_function`` must be a callable with one argument.rB  whatever.parquet)r#  r=   z0``name_function`` must produce unique filenames.c                    dS )Nr;  r   r  s    r"   r  zAtest_throws_error_if_custom_filename_is_invalid.<locals>.<lambda>j  s    2D r$   )	r   r   r   r   r   rM   r   rW   r   )r   r=   r   rZ  r   s        r"   /test_throws_error_if_custom_filename_is_invalidr=  \  s   	VB
,}}}55 C 
	+	+	+B	S
 
 
 K K 	b(:6JJJK K K K K K K K K K K K K K K
 
L
 
 
 U U 	b(D(DVTTTU U U U U U U U U U U U U U U U U Us$   BBB'CCCc                   t          |           }t          j        g dg dd          }t          j        |d          }|                    ||dgd d	           t          j        |          D ]?\  }}}|D ]}|d
v sJ 	|D ]+}	|	g d t          |j	                  D             ddR v sJ ,@t          j
        ||d          }
t          ||
ddd           d S )N)franklimarcelaluis)canadachina	venezuelarE  )
first_namecountryr   r0   rG  c                    |  dS )N-cool.parquetr   r  s    r"   r  z5test_custom_filename_with_partition.<locals>.<lambda>z  s    1 3 3 3 r$   F)r=   r  r#  r   )zcountry=canadazcountry=chinazcountry=venezuelac                    g | ]}| d S )rI  r   r   s     r"   r#   z7test_custom_filename_with_partition.<locals>.<listcomp>  s!    EEE!Q%%%EEEr$   r   r   r   )r   rM  r  )r   r   r   r   r   r   r   r\  r  r1   r   r   )r   r=   r   rZ  r   rE   dirsr   dirr]  r8  s              r"   #test_custom_filename_with_partitionrM  m  s   	VB
,<<<DDD	
 	
 C 
	+	+	+BMM
[33     '"++  4 	 	C      
  	 	D EEuR^/D/DEEE"       	 _Re<<<FVEU     r$   c                   |dk    rt           rt          j        d           dd lm} ddlm}m} t          j	        ddgddgd	          }| 
                    d
          }t          j        |d          }|                    ||dgd           | 
                    d          }t          j                            |          }	 ||	|dd |t          j        dt          j                    fg                               d }
t          j        ||          } |j        |                                          }t+           |
|           |
|          d           t          j        ||          } |j        |                                          }t+           |
|           |
|          d           d S )Nr2   z6fastparquet reads as int64 while pyarrow does as int32r   )HivePartitioningwrite_datasetrF   rI   r   r   )r  col2zfoo-daskr0   r  Fr  zfoo-pyarrowzpart.{i}.parquetr  )r   base_dirbasename_templateformatr  c                <    |                      d          ddg         S )NrQ  r  )r  r  s    r"   _prepz9test_roundtrip_partitioned_pyarrow_dataset.<locals>._prep  s    }}V$$ff%566r$   r   r  )r   rM   rG   pyarrow.parquetr  pyarrow.datasetrO  rP  r   r   r  r   r   r   r  r  r  r   r   r
  r  r   )r   r=   r  rO  rP  r   	dask_pathr   pa_pathr;  rV  df_read_dask
df_read_pas                r"   *test_roundtrip_partitioned_pyarrow_datasetr]    s    =MNNN      ???????? 
1vSz::	;	;B Z((I
.
+
+
+CNN9V6(PUNVVV ll=))GH  $$EM,%%bi&"(**1E0F&G&GHH   7 7 7 ?9V<<<Ly))3355JeeL!!55#4#4%HHHH ?76:::Lw''1133JeeL!!55#4#4%HHHHHHr$   filter_value)rF   )rV   r   rN   )idsc                   | dz  }t          j        g dg dd          }|                    ||           dd|fg}t          j        |||          }t          j        |||          }t          ||           t          j        |d	
          }| dz  }|                    ||           t          j        |||          }t          j        |||          }t          ||d           dS )<Regression test for https://github.com/dask/dask/issues/8720z$in_predicate_iterable_pandas.parquetr  r  r  r   r  rx  rs  rI   r0   z"in_predicate_iterable_dask.parquetFr  N)r   r   r   r   r   r   r   )	r   r=   r^  r   r   ri  rZ  r[  r   s	            r"   #test_in_predicate_can_use_iterablesrb    s    <<D	ll..
 
B MM$vM&&&T<()G_T&'BBBFtFGDDDHfh .
+
+
+C::DNN4N'''_T&'BBBFtFGDDDHfhE222222r$   c                   t          j        t          d          g ddd          }|                    | |           dgg}t          j        | ||          }t          j        | ||          }t          ||d	
           t          j        t          d          5  dgg}t          j        | ||           d d d            d S # 1 swxY w Y   d S )Nrw  )rF   rF   rI   rI   r  r  r   r   r  r   r0   r   )r  znot in)rF   rI   rs  Fr  z"not a valid operator in predicatesrB  )r  znot eqrF   )
r   r  r  r   r   r   r   rM   r   rW   )r   r=   r   ri  rZ  r[  unsupported_ops          r"   test_not_in_predicatere    sG   
,Ahh55566  C NN8FN+++'()G_XfgFFFFxHHHHfhE2222	z)M	N	N	N I I-./
HHHHI I I I I I I I I I I I I I I I I Is   CCC)r  rx  r*   )r  ru  r*   )zone-item-single-nestzone-item-double-nestztwo-item-double-nestztwo-item-two-nestc                   | dz  }t          j        g dg dd          }|                    ||           t          j        t
          d          5  t          j        |||           d	d	d	           n# 1 swxY w Y   t          j        |d
          }| dz  }|                    ||           t          j        t
          d          5  t          j        |||           d	d	d	           d	S # 1 swxY w Y   d	S )ra  zgh_8720_pandas.parquetr  r  r  r   zValue of 'in' filterrB  rs  NrI   r0   zgh_8720_dask.parquet)	r   r   r   rM   r   r  r   r   r   )r   r=   r^  r   r   r   s         r"   &test_in_predicate_requires_an_iterablerg    s   " ..D	ll..
 
B MM$vM&&&	y(>	?	?	? C C
V\BBBBC C C C C C C C C C C C C C C
 .
+
+
+C,,DNN4N'''	y(>	?	?	? C C
V\BBBBC C C C C C C C C C C C C C C C C Cs$   A77A;>A;C44C8;C8c                8   t          j        dt          d          i          }| dz  }|                    ||           t	          j        t          d          5  t          j        ||d          }d d d            n# 1 swxY w Y   t          ||           d S )	Nr   r*   z(test_deprecate_gather_statistics.parquetr   
deprecatedrB  T)r=   gather_statistics)
r   r   r  r   rM   rJ  rK  r   r   r   )r   r=   r   r   r   s        r"    test_deprecate_gather_statisticsrk    s    
 
sE"II&	'	'B@@DMM$vM&&&	m<	8	8	8 
 
o"
 
 

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 c2s   A??BBc                   t          |           }t          j        d          }t          j        d          }ddlm} |                    |j        f          d             }|                    ddgdd	gd
          }|                    |d          }|                    |           |	                    |          }t          ||           d S )Ncudf	dask_cudfr   )pyarrow_schema_dispatchc                4    |                                  j        S r8   )to_arrowr  )objs    r"   get_pyarrow_schema_cudfz>test_gpu_write_parquet_simple.<locals>.get_pyarrow_schema_cudf#  s    ||~~$$r$   abcdefr   r  r   r  )r   rM   r  dask.dataframe.dispatchro  registerr   	from_cudfr   r   r   )	r   r   rm  rn  ro  rs  r   r   rl  s	            r"   test_gpu_write_parquet_simplery    s    	VBv&&D#K00I??????%%t~&788% % 98% 
s	
 	

 
B 

b!
$
$CNN2

 
 
$
$Cb#r$   c                D   t          |           }d|  }ddi}t          j        dt          d          i          }t	          j        |d          }|                    |           |                    |d|	          }t          |j        d
          }|j	        sJ |j	        d         dk    sJ t	          j
        ||          }t          |j        d          }|j	        sJ |j	        d         dk    sJ |                    |d|	          }t          |j        d
          }|j	        rJ t	          j
        ||          }t          |j        d          }|j	        rJ t          j        d          5  |                    |d|	          }t          |j        d
          }|j	        sJ |j	        d         dk    sJ t	          j
        ||          }t          |j        d          }|j	        sJ |j	        d         dk    sJ 	 d d d            d S # 1 swxY w Y   d S )Nzsimplecache://target_protocolr]  r   r*   rI   r0   F)r   storage_optionsr  retriesr   )r|  r  )r}  )r   r   r   r  r   r   r   r   r  r   r   r  )	r   r   	remote_fnr|  r   r   scalarr  r   s	            r"   !test_retries_on_remote_filesystemr  3  s    
VB)))I(&1O	sE"II&	'	'B
.
+
+
+CNN2 ^^Iuo^VVFfk<00EY'1,,,,?9oFFFDdi00EY'1,,,, ^^B^OOFfk<00E    ?2???Ddi00E     
q	!	!	! 1 1uo   
 
 &+|44     +q0000y/JJJ$)^44     +q000001 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1s   6BHHHfsfsspecc                   ddl m} t          j        dt	          d          i          }t          j        |d                              | |           |p	 |            }t          j        | ||          }|Ht          t          |j        j                                                            j        j        }||u sJ t!          ||           d S )	Nr   r  r   r*   rI   r0   r   r=   
filesystem)r  r  r   r   r  r   r   r   r   r  r  r  r  r  io_funcr  r   )r   r=   r  r  r   r  r   layer_fss           r"   test_filesystem_optionr  a  s    <<<<<<	sE"II&	'	'BN21%%%00&0III(((J
/  C
 
zSX_33556677?B:%%%%c2r$   rg   c                   ddl m} ddlm} t	          j        dt          d          i          }|p	 |            }t          j        |d          	                    | |           t          j
        | d	|
          }t          t          |j        j                                                            j        j        }t%          ||          sJ t%          |j        |          sJ t'          ||           d S )Nr   )ArrowFSWrapperr  r   r*   rI   r0   )r  r5   r  )fsspec.implementations.arrowr  
pyarrow.fsr  r   r   r  r   r   r   r   r  r  r  r  r  r  r  r  r   )r   r  r  r  r   r   r  s          r"   test_pyarrow_filesystem_optionr  s  s    <;;;;;******	sE"II&	'	'B		   BN21%%%00b0III
/  C
 D//112233;>Hh/////hk?33333c2r$   c                 :    t          j        ddddid            d S )Nz$s3://coiled-data/uber/part.0.parquetr5   	anonymousT)r  r|  r  r  r   r$   r"   (test_pyarrow_filesystem_option_real_datar    s8     O.$d+	     r$   c                0   ddl m} d}t          | dz            }t          j        dt          d          i          }  |d          d	          }|                    |d
|           t          j        |d
|          }t          ||           |                    |d
|           t          t          |                                                     dk    s
J d            t          |                    |d                    dk    sJ |                    |d
d|           t          |                    |d                    dk    s
J d            t          j        |d
|          }t          |t          j        ||g                     d S )Nr   )get_filesystem_classz/read1write1r   r*   memoryF)use_instance_cacher5   r  zwrote to local fs)detailrF   T)r=   rZ   r  rI   zshould have two parts)r  r  r   r   r   r  r   r   r   r   rU   r   iterdirlsr  )r   r  key1key2r   r  r   r  s           r"   (test_fsspec_to_parquet_filesystem_optionr    s   ++++++Dx("##D	sE"II&	'	'B	'		h	'	'5	A	A	ABMM$yRM888 /  C
 c2NN4	bN999 tH$$&&''((A---/B--- ruuT%u(())Q.... NN4	$2NFFFruuT%u(())Q...0G...?4	bAAADdBIsCj))*****r$   c                   t          j        t          d          dgdz  d          }| dz  }|dk    rdnddi} |j        |f|d	d
| t	          j        t          d          5  t          j        ||dg          }t          ||           d d d            n# 1 swxY w Y   t	          j        t          d          5  t          j        ||dg          }t          ||           d d d            d S # 1 swxY w Y   d S )Nr*   rc  r   z#test_select_filtered_column.parquetr5   write_statisticsr  TFr   zSorted columns detectedrB  r  rs  r   r  N)
r   r   r  r   rM   rJ  r  r   r   r   r   r=   r   r   r  r   s         r"   test_select_filtered_columnr    s   	E"IIUGbL99	:	:B;;D#)Y#6#6GTJEBM$<vU<<e<<<	k)B	C	C	C  od6<N;OPPP"c               
k)B	C	C	C  od6<Q;RSSS"c                 s$   &)BB"B)C77C;>C;c                8   t          j        t          d          dgdz  d          }| dz  }|dk    rdnddi} |j        |fd	|i| t	          j        ||d
g          }t          ||           t	          j        ||dg          }t          ||           d S )Nr*   rc  r   z,test_select_filtered_column_no_stats.parquetr5   r  r  Fr=   r  rs  r  )r   r   r  r   r   r   r   r  s         r"   $test_select_filtered_column_no_statsr    s    	E"IIUGbL99	:	:BDDD#)Y#6#6GUKEBM$//v////
/$v8J7K
L
L
LCb#
/$v8M7N
O
O
OCb#r$   rW  z-dataframe.convert-string requires pandas>=2.0c                   t          j        g dg dg dd                              d          }| dz  }|                    ||           t          j                            d|i          5  t          j        ||          }d d d            n# 1 swxY w Y   |r7|	                    d	d
i          }|j
        	                    d
          |_
        n|}t          ||           |r|dk    rdnd}t          |j        j                  |k    sJ t          j                            d|i          5  t          j        |d          }d d d            n# 1 swxY w Y   t          j                            d| i          5  t          j        |d          }	d d d            n# 1 swxY w Y   |j        |	j        k    sJ d S )Nru  rt  ghir   rI   r  r  rP  rQ  out.parquetr   r  r  string[pyarrow]r2   rI   rF   r5   )r   r   r   r   r  rH  rV   r   r   r   r/   r   rU   r  r  )
r   rW  r=   r   outfiler   r[  n_layersrq  r   s
             r"    test_read_parquet_convert_stringr    s   
 
###)))///JJ
 
inn  &GMM'&M)))	4nE	F	F 6 6ogf5556 6 6 6 6 6 6 6 6 6 6 6 6 6 6  99c#4566!../@AAc8"Ev'>'>qqAHsx8++++ 
4nE	F	F : :wy999: : : : : : : : : : : : : : :	4.6HI	J	J : :wy999: : : : : : : : : : : : : : : :######s6   /BBB4EEEF''F+.F+c           	        t          j        t          j        g dd          t          j        g dd          t          j        g dd          t          j        g dd          d	                              d
          }| dz  }|                    ||           t          j                    t          j                    i}t          j	        
                    ddi          5  t          j        | ddd|j        i          }ddd           n# 1 swxY w Y   |                    dt          j                    t          j                    d          }|j                            d          |_        t#          ||           dS )zcMake sure that when convert_string, dtype_backend and types_mapper are set,
    all three are used.r  r)  rr   r  r!  )g?g333333@g @Float32r  )r  r  rQ  r  r  r  r   r  Tr5   rQ  rR  rS  Nr  rP  )r   r   r+  r   r   r  r  Float64Dtyper  rH  rV   r   r   rW  r   
Int64Dtyper/   r   )r   r=   r   r  rR  r   r[  s          r"   0test_read_parquet_convert_string_nullable_mapperr    s    
000AAA999G444???)<<<???(;;;		
 	

 
 inn  &GMM'&M))) 	
bo''L 
4d;	<	< 
 
o*+\-=>	
 
 

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 yy"""	
 	
 H ^**+<==HNc8s   ( DDDrQ  c                T    |dk    rdnd}t          j        t          j        ddt           j        ddgd| 	          t          j        d
t           j        dd
dgd| 	          t          j        dddt           j        dgd| 	          t          j        ddddt           j        gd| 	          d          }t	          j        |d          }t          j         fd            |                                }t          j	        fdt          |          D                        |dk    rMt          j        t          d          5  t	          j         ||           ddd           dS # 1 swxY w Y   dS t	          j         ||          }t          ||d            dS )!r1  rQ  r2  r3  rF   rI   r  r   r!  rr   TFr"  r#  r$  r%  r&  r'  r   r   r(  r  r)  r*  r0   c                    t           j                            |                               i           }t	          j        |d| dz             dS r5  r8  r:  s      r"   r<  z+test_dtype_backend.<locals>.write_partition3  r=  r$   c                .    g | ]\  }} ||          S r   r   r?  s      r"   r#   z&test_dtype_backend.<locals>.<listcomp><  rA  r$   r2   z `dtype_backend` is not supportedrB  r=   r.  Nr  )r   r   r+  r,  r   r   r  rE  rF  r   rG  rM   r   rW   r   r   rL  s	   `       @r"   test_dtype_backendr    s<    &)999""{K	Aq"%A.6Kk6K6KLLLrueT519P;9P9P   Cc25#6>U>U>UVVVCc36>T{>T>TUUU	
 	
	
 	
B .
+
+
+C	\> > > > \> !!JLJJJJIj4I4IJJJKKK ]:-OPPP 	R 	ROHV=QQQQ	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R
 xmTTT"d......s   E22E69E6z*pd.Index does not support int32 before 2.0c           	     N   t          j        ddgddgd          }| dz  }|                    |ddg	           t          j        |d
          }t          j        ddgt          j        t          j        ddgd                    dddg          }t          ||           d S )NrF   rI   r'   r(   r   r  r5   r   )r=   r)  r   r   rr   r  r   r.   )r   r   r   r   r   rj  r  r   )r   r   outdirr   r[  s        r"   3test_read_parquet_preserve_categorical_column_dtyper  I  s    
 
QF#s44	5	5B%FMM&C5MAAA
/&
3
3
3C|Cjr~bh1vW.M.M.MNNOO!f  H c8r$   zRequires pd.ArrowDtypec                &   t          j        t          j        ddgd          ddgd          }| dz  }|                    |d	
           t	          j        |d	d	          }t          j        |d	d	          }t          ||t                     d S )Nr'   r(   r`  rr   rF   rI   r   r  r5   r   r  )sort_results)r   r   r+  r   r   r   r   r   )r   r   r  r   rZ  s        r"   test_dtype_backend_categoricalsr  [  s     
BIsCj
CCC1a&QQ	R	RB%FMM&M+++
/&)
L
L
LC
/&)
L
L
LCc3]333333r$   ri  )r   rt  r  c                   ddl m} t          j        t	          d          ddgdz  d          }t          j        |d	          }|                    | d
gdd           t          j	        d
t          j
                    fg          }t          d|          }t          j        | d|i|d          }t          j        |  |di ||d          }t          ||d           |d
         j        dk    sJ d S )Nr   )r  r_  rc  r  r  r   rI   r0   r   Fr5   )r   r  r   r=   r  r  r  )r  ri  r=   )r  ri  r=   r  r`  r   )rX  r  r   r   r  r   r   r   r  r  r)  r  r   r   rs   )	r   ri  pd_partitioningr  rq  r  r  r   rZ  s	            r"   )test_non_categorical_partitioning_pyarrowr  g  s7    @?????
,U3ZZuenr.ABB
C
CC>#1---DOO3%U9     Ybikk*+,,Fvf555L
/.	  C /$_44|44	  C c3E****s8>Z''''''r$   (  
__future__r   r  r  r  r   rR  decimalr   unittest.mockr   numpyr   r/  r   rM   packaging.versionr   r  r  dask.dataframer  r   rD  dask.array.numpy_compatr   dask.blockwiser   r	   dask.dataframe._compatr
   r   r   dask.dataframe.io.parquet.corer   dask.dataframe.io.parquet.utilsr   dask.dataframe.optimizer   dask.dataframe.utilsr   r   dask.layersr   
dask.utilsr   dask.utils_testr   r2   r  r  ImportErrorr5   r  rX  rW  r  r  SKIP_FASTPARQUETrY   skipifrQ   SKIP_PYARROWSKIP_PYARROW_REASONrR   nrowsr1   r   r  r  r   r   r   fixturer:   r=   rc   fp_pandas_xfailri   rn   r[   r   r   r   r   r   r   r   r   r   r   r   r   r
  r  r  r  r  r-  rO  r\  rg  ru  rw  r  r  r  r  r  r  r  	Timedeltar  r  r  r  skip_with_pyarrow_stringsr  r  r  binaryrj  r   mapr  r   r  rG   r  xfail_with_pyarrow_stringsr  r  r  r  r1  rC  rK  rN  rZ  r^  rb  ro  r~  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r"  r%  r*  r-  r8  r<  rE  rN  rU  r[  r^  r`  rf  rp  rt  ry  r{  r~  r  r  r  r  r   permutationr  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r  r  r  r"  r$  r*  r0  r7  r9  ri  rm  ry  r~  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r%  r/  r9  r=  rM  r]  rb  re  rg  rk  gpury  r  r  r  slowr  r  r  r  r  r  r  r  r  r  r   r$   r"   <module>r     s(   " " " " " "       				        # # # # # #          4 4 4 4 4 4            . . . . . . 8 8 8 8 8 8 8 8 N N N N N N N N N N 5 5 5 5 5 5 B B B B B B > > > > > > C C C C C C C C ( ( ( ( ( ( ' ' ' ' ' ' % % % % % %A
 (-(?@@	  - - -K'-,,-)#mBN33OO ) ) )	B#mC((OOO)          	BBB #? ;%%S &    v) {!!,7J!KK
R\..u...,,uuU||,,,  "(11EE%LL111	
B
B
B   bnR[111 ]*:;;;Yl333      D %$&& 5 5 5 : : : %773 3 87 3@ 5$-00	 	  10	 	 	 	   (   " 
C 
C 
C   . " " "J R R RjV V V ) ) )X " " ";" ;" ;"| + + +  S S S      , +$$!!*S %  	
 	
 	
 /7 /7  /7d      > , , ,: 5$-88  98@  2- - -`7 7 7 %//5$-007 7 10 0/ 77 7 7&5 5 56 5$-88' ' 98'@ 5$-88
 BM,3???BLa   		
 
q#q	!	!3' L L  98L2  8 5$-883 3 983, &  '&> 0 0 08 8 80 !	sIII&	'	'R0	sOOO,	-	-r26	s.../	0	0"b9BL#111233 '3UC3E3EQUVV	
 BL#~r~ooo>>?@@C5!	

 
sNBN999556	7	7lSE=RS	sDDR\3E3E3E!F!FGGH	I	I2rR	s.../	0	0	7	7	A	A2rJ	sIII&	'	'	.	.x	8	8"bA	sIII&	'	'	.	.x	8	8"bA	sIII&	'	'	.	.x	8	8"bA	s.../	0	0	7	78H	I	I2rR	s.../	0	0	7	78M	N	NPRTVW	s.../	0	0	7	78M	N	NPRTVW	sIII&	'	'	.	.x	8	8"bA	sIII&	'	'	.	.y	9	92rB	sIII&iii	8	8	8"bA	sIII&hbhyyyu.M.M.M	N	N	NPRTVW	IIIIII66	7	7R@	IIIIII66c
	K	K	KRQST	sIII&	'	'R0	sLLL)	*	*B3	s,,,-	.	.B7	s,,,-	.	.B7	s,,,-	.	.B7C"% %L &04 04 '&M% %N04f K/MM(4K4K"K]   "? "?	 "?J '< < ('<$? ? ?  )D%=99  :9 $ $  $ &TQ TQ '& TQn 5$-00GY#788  98 10 B ) ) )" + + +D D D6  8 $88F F 98 F. " " "* - - -` 4 4 4n5 5 54!8 !8 !8H"& "& "&J / / /< / / /6 @AA6 6 BA 6: y+&>??0 0 @?0  UDM226 6 32 6    G G G,* * **
 
 
  : )A)A)ABB
/ 
/ CB
/ )A)A)ABB/ / CB/"  !%!")#*	  !%")#*	  $W&!	
 	
, !%")#*	  !%!")#*	  $W&!	
 	
, #'!+W 5 "*#,  #& $")#*  #6 $!")#*   22&7	
 	
QFH H HR SH HR% % %"A6 A6 A6H      '( '( '(T'( '( '(T#B #B #BLG G G- - -*F F F*I I I  , , ,$, , ,< < < c---.//c???+,,c---.//\R\3(G(G"HIIJJ\R\3yyy(A(A"BCCDDc44BL2M2M2M N NOOPQQc44BL2D2D2D E EFFGHHc---.//66x@@BL#yyy)**11(;;+##O/MM,4O4O"OE $  	
 	
 	
 	BL#yyy)**11(;;+##O/MM,4O4O"OE $  	
 	
 	
 	c999%&&--h77c999%&&--i88c999%YYY777$$$%XRX6H6H6Hu-U-U-U	
 	
 	
 	999999556699999955SzJJJc999%&&c<<<())c+++,--c+++,--c+++,--I%( (R S( ( T  	 	 	 
Y 
Y 
Y 	0 	0 	0 0 0 0 tUm444-884 4  98 544    GT?33H H 43H(< < < )D%=994)>)>t)D)D"EFF@ @ GF :9@8    $     *  4/ / / ( ( (V $! $! $!N +aW55.u>>R R ?> 65 R< +aW55+ + 65 +> &&+8		
 	
 	
 	77;  +dE];;5 5 <;  50 +dE];;; ; <; ;6   :	/ 	/ 	/  2 	- 	- 	- dE]33$55tVn55:H :H  65 65 43 
:Hz *S#J77(C (C 87 (CV dE]33&@&@&@AA3) 3) BA 43 3)l dE]33&D&D&DEE=4 =4 FE 43 =4@   ( 	. 	. 	.( ( (, .t==% %  >=%6 - - -$ SQ SQ SQl 0 0 0  e_---.  1  1	  1F Wvuo G G	  
G@ G G G,   6  $  & 	" 	" 	"% % %*4 4 4    u66    76  & t[&9::$ $ ;: $N C C C2 +dE];;  <; 6* * *< "7 "7 "7J # # #L $5559 59  65 59p* * *8 $  $  $F  .	O 	O 	O4 4 4,@ @ @) ) )@ 1 1 1(# # #'4 '4 '4T .0C0C0CDD$ $ ED$N .u>>-1v66  76 ?>D 55A A 65 AH	 	 	T T TP P P*(7 (7 (7VU U U"! ! !H *I *I *IZ 1#sD)9?WXX3 3 YX3,I I I& 	
		)
/	*+		   C C C&  "   , *1 *1 *1Z $/00  10" //  0/ $     + + +B  
 
 
 )D%=99M   $ $  :9$< M   $ $  $N +;Y*GHH%.WXX%/ %/ YX IH %/P J       %.FGG4 4 HG 4 T-?,@+A$BCC( ( DC ( ( (s6   B! !B65B6:C C$#C$(C/ /C98C9