
    o[we                      |   d Z ddlZddlmZ ddlmZ ddlZddlZddlZ	ddl
Z
ddlmZmZ ddlmZ ddlmZmZmZmZmZ ddlZddlmZ ddlmZ dd	lmZmZm Z m!Z!m"Z" 	 ddl#Z#d
Z$n# e%$ r dZ$Y nw xY w	 ddl&Z&d
Z'n# e%$ r dZ'Y nw xY we
j(        )                    d          Z* e
j+         e
j,        de
j(        -                    e' p ed          dk    d                     e
j,        de
j(        -                    e$ d                    g          d             Z.e
j+        d             Z/e
j+        d             Z0e
j+        d             Z1e
j+        d             Z2e
j+        d             Z3 e
j+        ej        4                    ej5        j6                  ej        4                    ej5        j7                  ej        4                    ej5        j8                  ej        9                    dd          ej        9                    dd          ej        9                    dd          ej        9                    d d          g          d!             Z:	 	 	 	 	 	 	 	 	 d5d#Z;d$ Z<d% Z=d& Z>d' Z?d( Z@d) ZAd* ZBd+ ZCd, ZD G d- d.          ZE G d/ d0eE          ZF G d1 d2eE          ZG G d3 d4eE          ZHdS )6z test parquet compat     N)Decimal)BytesIO)
get_optionusing_copy_on_write)is_platform_windows)pa_version_under7p0pa_version_under8p0pa_version_under11p0pa_version_under13p0pa_version_under15p0)Version)FastParquetImplPyArrowImpl
get_engineread_parquet
to_parquetTFz2ignore:DataFrame._data is deprecated:FutureWarningfastparquetmode.data_managerarrayz4fastparquet is not installed or ArrayManager is usedreason)markspyarrowpyarrow is not installed)paramsc                     | j         S Nparamrequests    <lib/python3.11/site-packages/pandas/tests/io/test_parquet.pyenginer#   <   s    $ =    c                  <    t           st          j        d           dS )Nr   r   )_HAVE_PYARROWpytestskip r$   r"   par*   Q   s      0.///9r$   c                      t           st          j        d           n't          d          dk    rt          j        d           dS )Nzfastparquet is not installedr   r   z.ArrayManager is not supported with fastparquetr   )_HAVE_FASTPARQUETr'   r(   r   r)   r$   r"   fpr-   X   sI     F23333	'	(	(G	3	3DEEE=r$   c                  4    t          j        g ddd          S )N         fooAB)pd	DataFramer)   r$   r"   	df_compatr9   a   s    <iiie44555r$   c            
          t          j        t          d          t          t          dd                    t	          j        ddd          g dt          j        d	d
          d          } | S )Nabcr0            @      @float64dtypeTFT20130101r2   periods)abdef)r7   r8   listrangenparange
date_range)dfs    r"   df_cross_compatrQ   f   so    	eeAqkk""39555$$$z1555
	
 
	

 
B Ir$   c                     t          j        t          d          dt          j        dgg dg dg dt          t          dd                    t          j        d	d
                              d          t          j        ddd          dt          j        dgg dt          j        dd	          t          j	        d          t           j
        t          j	        d          gd          S )Nr;   rF   crF   NrS   )   foo   bars   bazr3   barbazr0   r<   r2      u1r=   r>   r?   r@          @      @rB   rC   rD   20130103)stringstring_with_nanstring_with_nonebytesunicodeintuintfloatfloat_with_nanbooldatetimedatetime_with_nat)r7   r8   rK   rM   nanrL   rN   astyperO   	TimestampNaTr)   r$   r"   df_fullro   x   s    <5kk #RVS1 0 0 0---,,,a$$IaOO**400YsCy999"BFC0'''j!<<<Z((Z(("	
 	
  r$   z2019-01-04T16:41:24+0200z%Y-%m-%dT%H:%M:%S%zz2019-01-04T16:41:24+0215z2019-01-04T16:41:24-0200z2019-01-04T16:41:24-0215c                     | j         S r   r   r    s    r"   timezone_aware_date_listrq      s     =r$   r1   c
                      pddipi  |r
|d<   |d<    fd}
9t          j                    5  |
|	           ddd           dS # 1 swxY w Y   dS  |
|	           dS )a  Verify parquet serializer and deserializer produce the same results.

    Performs a pandas to disk and disk to pandas round trip,
    then compares the 2 resulting DataFrames to verify equality.

    Parameters
    ----------
    df: Dataframe
    engine: str, optional
        'pyarrow' or 'fastparquet'
    path: str, optional
    write_kwargs: dict of str:str, optional
    read_kwargs: dict of str:str, optional
    expected: DataFrame, optional
        Expected deserialization result, otherwise will be equal to `df`
    check_names: list of str, optional
        Closed set of column names to be compared
    check_like: bool, optional
        If True, ignore the order of index & columns.
    repeat: int, optional
        How many times to repeat the test
    compressionNr#   c                     t          |           D ]D} j        fi 
 t          fi 	}dv r
d j        d<   t	          j        |           Ed S )Nr`   r0   r`   )check_names
check_likecheck_dtype)rL   r   r   loctmassert_frame_equal)repeat_actualrx   rw   rv   rP   expectedpathread_kwargswrite_kwargss      r"   comparez!check_round_trip.<locals>.compare   s    v 	 	ABM$//,///!$66+66F H,,5912!'%'    	 	r$   )rz   ensure_clean)rP   r#   r   r   r   r   rv   rw   rx   r|   r   s   ` ```````  r"   check_round_tripr      s   D  8M4#8L#K '!'X &H            |_ 	$GFOOO	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	s   AA!$A!c                 P   t           rjddlm} |                    | d          }t	          |j        j                  t	          |          k    sJ |j        j        t          |          k    sJ dS ddlm	} |	                    | d          }|j
        j        j        |k    sJ dS )zCheck partitions of a parquet file are as expected.

    Parameters
    ----------
    path: str
        Path of the dataset.
    expected: iterable of str
        Expected partition names.
    r   NF)validate_schemahive)partitioning)r   pyarrow.parquetparquetParquetDatasetlen
partitionspartition_namessetpyarrow.datasetdatasetr   schemanames)r   r   pqr   dss        r"   check_partition_namesr      s      
=$$$$$$##D%#@@7%566#h--GGGG!1S]]BBBBBB$$$$$$**T*77#*0H<<<<<<r$   c                     d}t          j        t          |          5  t          | dd           d d d            d S # 1 swxY w Y   d S )Nz.engine must be one of 'pyarrow', 'fastparquet'matchr3   rX   )r'   raises
ValueErrorr   )r9   msgs     r"   test_invalid_enginer      s    
:C	z	-	-	- 2 2E51112 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2s   =AAc                     t          j        dd          5  t          |            d d d            d S # 1 swxY w Y   d S )Nio.parquet.enginer   r7   option_contextr   )r9   r*   s     r"   test_options_pyr      s     
	.		:	: $ $###$ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $   377c                     t          j        dd          5  t          |            d d d            d S # 1 swxY w Y   d S )Nr   r   r   )r9   r-   s     r"   test_options_fpr     s     
	.	>	> $ $###$ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $r   c                     t          j        dd          5  t          |            d d d            d S # 1 swxY w Y   d S )Nr   autor   )r9   r-   r*   s      r"   test_options_autor     s     
	.	7	7 $ $###$ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $r   c                 4   t          t          d          t                    sJ t          t          d          t                    sJ t	          j        dd          5  t          t          d          t                    sJ t          t          d          t                    sJ t          t          d          t                    sJ 	 d d d            n# 1 swxY w Y   t	          j        dd          5  t          t          d          t                    sJ t          t          d          t                    sJ t          t          d          t                    sJ 	 d d d            n# 1 swxY w Y   t	          j        dd          5  t          t          d          t                    sJ t          t          d          t                    sJ t          t          d          t                    sJ 	 d d d            d S # 1 swxY w Y   d S )Nr   r   r   r   )
isinstancer   r   r   r7   r   )r-   r*   s     r"   test_options_get_enginer     s   j++[99999j//AAAAA		.		:	: F F*V,,k:::::*Y//=====*]33_EEEEEEF F F F F F F F F F F F F F F
 
	.	>	> F F*V,,o>>>>>*Y//=====*]33_EEEEEEF F F F F F F F F F F F F F F
 
	.	7	7 F F*V,,k:::::*Y//=====*]33_EEEEEEF F F F F F F F F F F F F F F F F Fs9   A-CCC8A-E22E69E6A-HHHc                     ddl m}  |                     d          }|                     d          }t          sdn)t	          t
          j                  t	          |          k     }t          sdn)t	          t          j                  t	          |          k     }t          o| }t          o| }|s&|s%|rId| d}t          j
        t          |          5  t          d	           d d d            n# 1 swxY w Y   nDd
}t          j
        t          |          5  t          d	           d d d            n# 1 swxY w Y   |rKd| d}t          j
        t          |          5  t          d	           d d d            d S # 1 swxY w Y   d S d}t          j
        t          |          5  t          d	           d d d            d S # 1 swxY w Y   d S d S d S )Nr   )VERSIONSr   r   FzPandas requires version .z. or newer of .pyarrow.r   r   z%Missing optional dependency .pyarrow.z. or newer of .fastparquet.z)Missing optional dependency .fastparquet.)pandas.compat._optionalr   getr&   r   r   __version__r,   r   r'   r   ImportErrorr   )r   
pa_min_ver
fp_min_verhave_pa_bad_versionhave_fp_bad_versionhave_usable_pahave_usable_fpr   s           r"   "test_get_engine_auto_error_messager   '  s    100000 i((Jm,,J 	@W())GJ,?,??  !	D[,--
0C0CC  #>+>'>N&B/B+BN #. # 	#S
SSSE{%888 # #6"""# # # # # # # # # # # # # # # <E{%888 # #6"""# # # # # # # # # # # # # # #  	#W
WWWE{%888 # #6"""# # # # # # # # # # # # # # # # # # @E{%888 # #6"""# # # # # # # # # # # # # # # # # ### # # #sH   C00C47C4D55D9<D9#F  FF*GGGc                 @   | }t          j                    5 }|                    ||d            t          ||          }t          j        ||           t          ||ddg          }t          j        ||ddg                    d d d            d S # 1 swxY w Y   d S N)r#   rs   r#   rF   rH   )r#   columnsrz   r   r   r   r{   rQ   r*   r-   rP   r   results         r"   test_cross_engine_pa_fpr   R  s     
B			 6d
d24888d2...
fb)))d2SzBBB
fb#sn5556 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6   A0BBBc                 @   | }t          j                    5 }|                    ||d            t          ||          }t          j        ||           t          ||ddg          }t          j        ||ddg                    d d d            d S # 1 swxY w Y   d S r   r   r   s         r"   test_cross_engine_fp_par   `  s    	B			 6d
d24888d2...
fb)))d2SzBBB
fb#sn5556 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6r   c                   h    e Zd Zd Zd Zej        j        ej        j        d                         Z	dS )Basec                     t          j                    5 }t          j        ||          5  t	          |||d            d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nr   rs   )rz   r   r'   r   r   )selfrP   r#   excerr_msgr   s         r"   check_error_on_writezBase.check_error_on_writen  s    _ 	?$s'222 ? ?2tV>>>>? ? ? ? ? ? ? ? ? ? ? ? ? ? ?	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	?s3   A#AA#A	A#A	A##A'*A'c                     t          j                    5 }t          j        |          5  t          |||d            d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nr   )rz   r   external_error_raisedr   )r   rP   r#   r   r   s        r"   check_external_error_on_writez"Base.check_external_error_on_writet  s    _ 	?$)#.. ? ?2tV>>>>? ? ? ? ? ? ? ? ? ? ? ? ? ? ?	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	?s3   A!A	A!	A	A!A	A!!A%(A%c                 F   |dk    rt          j        |           t           |dddd          d          5 }|                    |                                           t          |j                  }d d d            n# 1 swxY w Y   t          j        ||           d S )	Nr   iodatar   zsimple.parquetrb)mode)content)	r'   importorskipopenserve_contentreadr   urlrz   r{   )r   
httpserverdatapathr9   r#   rJ   rP   s          r"   test_parquet_read_from_urlzBase.test_parquet_read_from_urlz  s     V'''((44DEEDQQQ 	.UV$$QVVXX$666jn--B	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	b),,,,,s   =BBBN)
__name__
__module____qualname__r   r   r'   marknetwork
single_cpur   r)   r$   r"   r   r   m  sc        ? ? ?? ? ? [[- -  - - -r$   r   c                   >   e Zd Zd Zd Zej                            dg d          d             Zd Z	d Z
d Zd	 Zd
 Zd Zd Zd Zd Zd Zd Zej                            ed          d             Zej                            dg d          d             ZdS )	TestBasicc                     t          j        g d          ddt          j        d          t          j        g d          fD ]!}d}|                     ||t          |           "d S )Nr/   r0   r3   rC   z+to_parquet only supports IO with DataFrames)r7   Seriesrm   rM   r   r   r   )r   r#   objr   s       r"   
test_errorzTestBasic.test_error  s{    Iiii  L$$HYYY
 	D 	DC @C%%c6:sCCCC	D 	Dr$   c           	          t          j        t          d          t          t          dd                    d          }ddg|_        t          ||           d S )Nr;   r0   r<   r_   rd   r3   rX   )r7   r8   rK   rL   r   r   )r   r#   rP   s      r"   test_columns_dtypeszTestBasic.test_columns_dtypes  sR    \T%[[eAqkk9J9JKKLL U^
V$$$$$r$   rs   )Ngzipsnappybrotlic                 ^    t          j        dg di          }t          ||d|i           d S )Nr5   r/   rs   r   r7   r8   r   )r   r#   rs   rP   s       r"   test_compressionzTestBasic.test_compression  s:    \3			*++V=+2NOOOOOOr$   c           	          t          j        t          d          t          t          dd                    d          }t          j        dt          d          i          }t	          |||ddgi           d S )Nr;   r0   r<   r   r_   r   r   r   r7   r8   rK   rL   r   )r   r#   rP   r   s       r"   test_read_columnszTestBasic.test_read_columns  sz    \T%[[eAqkk9J9JKKLL<4;; 788	H:7N	
 	
 	
 	
 	
 	
r$   c           
          t          j        t          t          d                    t          d          d          }t          j        dddgi          }t	          ||||ddgid	gdgd
d           d S )Nr<   aabb)rd   partrd   r   r0   partition_colsr   )r   ==rF   )filtersr   )r   r   r   r   r|   r   )r   r#   tmp_pathrP   r   s        r"   test_read_filterszTestBasic.test_read_filters  s    \E!HH~~V 
 
 <A00*VH5%8$9ugNN	
 	
 	
 	
 	
 	
r$   c                 0   |dk    }|r>|dk    r8|j                             t          j                            d                     t          j        dg di          }t          ||           g dt          j        dd	          t          d
          g dg}|D ]T}||_
        t          |t
          j                  r|j
                            d           |_
        t          |||           Ug d|_
        d|j
        _        t          ||           d S )Nr   zfastparquet write into indexr   r5   r/   )r1   r2   r<   rC   r2   rD   r;   )r0   r2   r<   )rv   )r   r0   r1   r3   )node
add_markerr'   r   xfailr7   r8   r   rO   rK   indexr   DatetimeIndex
_with_freqname)r   r#   r   r!   rv   rP   indexesr  s           r"   test_write_indexzTestBasic.test_write_index  s5   - 	6]#:#:L##!!)G!HH   \3			*++V$$$ IIM*a000KKII	
  	B 	BEBH%!122 58..t44R[AAAAA 99V$$$$$r$   c                     |}t          j        dg di          }t           j                            g d          }||_        t          ||           d S )Nr5   r/   )rF   r0   )rF   r1   )rG   r0   )r7   r8   
MultiIndexfrom_tuplesr  r   )r   r*   r#   rP   r  s        r"   test_write_multiindexzTestBasic.test_write_multiindex  sX    \3			*++))*H*H*HIIV$$$$$r$   c           	         |}t          j        ddd          }t          j        t          j                            d                              dt          |          z  df          t          d                    }t           j	        
                    d	d
g|gddg          }|                    d           }||fD ]8}||_        t          ||           t          ||dddgi|ddg                    9d S )Nz01-Jan-2018z01-Dec-2018MS)freqr1   r2   ABCr   Level1Level2leveldate)r   r   r5   r6   r   r   )r7   rO   r8   rM   randomdefault_rngstandard_normalr   rK   r  from_productcopyr  r   )r   r*   r#   datesrP   index1index2r  s           r"   test_multiindex_with_columnsz&TestBasic.test_multiindex_with_columns  s   m]FFF\I!!!$$44a#e**na5HIIKK
 
 
 ++!5)'61B , 
 
 4((f% 	 	EBHR(((FS#J(?"cSVZ.    		 	r$   c                    t          j        g dg dd          }d dd}|                    d          }t          ||||           t          j        g dg ddg d	
          }t          ||||           g dg dg}t          j        t	          t          d                    d t          d          D             d|
          }|                    d          }t          ||||           d S )Nr/   )qrs)rF   rG   F)rs   r  T)dropr   r   )zyxwvutsrr  rX   rX   rY   rY   r3   r3   quxr-  onetwor/  r0  r/  r0  r/  r0     c                     g | ]}| S r)   r)   ).0is     r"   
<listcomp>z7TestBasic.test_write_ignoring_index.<locals>.<listcomp>  s    +A+A+A1QB+A+A+Ar$   )r/  r0  )r7   r8   reset_indexr   rK   rL   )r   r#   rP   r   r   arrayss         r"   test_write_ignoring_indexz#TestBasic.test_write_ignoring_index  s=    \			@@AA'+e<< >>t>,,V,RRRR \))///22:O:O:O
 
 
 	V,RRRR EDDDDD
 \qNN+A+Aa+A+A+ABB&
 
 
 >>t>,,V,RRRRRRr$   c                 N   t           j                            g d          }t          j        t          j                            d                              d          |          }|dk    r|                     ||t          d           d S |dk    rt          ||           d S d S )Nr  r1   )r<   r2   r  r   Column name must be a stringr   )r7   r  r  r8   rM   r  r  r  r   	TypeErrorr   )r   r#   
mi_columnsrP   s       r"   test_write_column_multiindexz&TestBasic.test_write_column_multiindex  s    ]../M/M/MNN
\I!!!$$44V<<j
 
 
 ]""%%FI'E     y  R((((( ! r$   c                 8   g dg dg}t          j        t          j                            d                              d          |          }ddg|j        _        |dk    r|                     ||t          d	           d S |d
k    rt          ||           d S d S )Nr,  )r0   r1   r0   r1   r0   r1   r0   r1   r1   r1  r1  r  r  r  r   zColumn namer   )r7   r8   rM   r  r  r  r   r   r   r   r   r   r#   r7  rP   s       r"   &test_write_column_multiindex_nonstringz0TestBasic.test_write_column_multiindex_nonstring  s    
 EDD$$$
 \I!!!$$44V<<f
 
 
 %h/
]""%%b&*mLLLLLy  R((((( ! r$   c                     |}g dg dg}t          j        t          j                            d                              d          |          }ddg|j        _        t          ||           d S )Nr,  r.  r1   r?  r  	ColLevel1	ColLevel2)	r7   r8   rM   r  r  r  r   r   r   r   r*   r#   r7  rP   s        r"   #test_write_column_multiindex_stringz-TestBasic.test_write_column_multiindex_string0  s      EDDDDD
 \I!!!$$44V<<f
 
 
 (5
V$$$$$r$   c                     |}g d}t          j        t          j                            d                              d          |          }d|j        _        t          ||           d S )N)rX   rY   r3   r-  r1   r1  r<   r  	StringCol)	r7   r8   rM   r  r  r  r   r  r   rE  s        r"   test_write_column_index_stringz(TestBasic.test_write_column_index_stringA  sp      .--\I!!!$$44V<<f
 
 
 &
V$$$$$r$   c                    g d}t          j        t          j                            d                              d          |          }d|j        _        |dk    r|                     ||t          d           d S t          ||           d S )Nr0   r1   r2   r<   r1   rH  r  NonStringColr   r:  )r7   r8   rM   r  r  r  r   r  r   r;  r   r@  s       r"   !test_write_column_index_nonstringz+TestBasic.test_write_column_index_nonstringO  s     \I!!!$$44V<<f
 
 
 )
]""%%FI'E     R(((((r$   zminimum pyarrow not installedr   c                    dd l m} |dk    r:t          j                            d          }|j                            |           t          j        t          j	        g dd          t          j	        g dd          t          j	        g d          t          j	        g d	          t          j	        g d
d          t          j	        g dd          t          j	        g dd          d          }t          j                    5 }|                    ||           t          ||          }t          ||d          }d d d            n# 1 swxY w Y   |d         j        t          j        d          k    sJ t!          j        t!          j	        g dd          t!          j	        g dd          t!          j	        g dd          t!          j	        g d	d          t!          j	        g d
d          t!          j	        g dd          t!          j	        g dd          d          }	|dk    r.|                    dd          }|	                    dd          }	t          j        ||	           d S )Nr   r   z.Fastparquet nullable dtype support is disabledr   r0   r1   r2   Nint64uint8)rF   rG   rS   N)TFTNrL  )      ?r\   r]   Nfloat32r?   )rF   rG   rS   rH   rI   rJ   gr   numpy_nullabler#   dtype_backendrF   Int64r@   UInt8r_   booleanFloat32Float64rS   r0   )axis)r   r   r'   r   r  r  r  r   tabler   rz   r   write_tabler   rA   rM   r7   r8   r&  r{   )
r   r#   r!   r   r   r_  r   result1result2r   s
             r"   test_dtype_backendzTestBasic.test_dtype_backend_  s   $$$$$$]"" ;$$G %  D L##D)))]???G<<]???G<<]#8#8#899]#<#<#<==]<<<99]#8#8#8)DD]#8#8#8)DD
 

 
 _ 	X$NN5$'''"4777G"4FVWWWG		X 	X 	X 	X 	X 	X 	X 	X 	X 	X 	X 	X 	X 	X 	X s|!RXi%8%88888<XoooW===XoooW===X3338DDDX777yIIIXlll':::X3339EEEX3339EEE 

 

 ]"" ll3Ql//G}}Sq}11H
gx00000s   :EEErA   )	rY  rZ  r[  objectzdatetime64[ns, UTC]rf   z	period[D]r]  r_   c                     t          j        dt          j        g |          i          }d }|dk    r*t          j        dt          j        g d          i          }t          ||ddi|           d S )Nvaluer@   rf   r]  rX  rV  r  )r7   r8   r   r   )r   r*   rA   rP   r   s        r"   test_read_empty_arrayzTestBasic.test_read_empty_array  s      \"E222
 
 G|RXb	::: H
 	2B Ch	
 	
 	
 	
 	
 	
r$   N)r   r   r   r   r   r'   r   parametrizer   r   r   r	  r  r!  r8  r=  rA  rF  rJ  rN  skipifr   rc  rg  r)   r$   r"   r   r     s       	D 	D 	D% % % [],N,N,NOOP P POP
 
 

 
 
&% % %8% % %  &S S S@) ) )) ) )"% % %"% % %) ) )  [+4STT/1 /1 UT/1b [
	
 
	
 
	
 
 
 
 
 
r$   r   c                      e Zd Zd Zd Zd Zd Zd Zd Zd Z	e
j                             e            d	          e
j                            e d
	          e
j                            deej        g          d                                     Zd Ze
j        j        d             Ze
j        j        d             Ze
j        j        e
j                            ddgg g          d                         Zd Zd Zd Zd Ze
j                            ded gddg          d             Zd Zd Z d Z!d Z"d  Z#d! Z$d" Z%d# Z&d$ Z'd% Z(d& Z)d' Z*d( Z+d) Z,d* Z-e
j                            e.d+	          d,             Z/d- Z0d.S )/TestParquetPyArrowc                     |}t          j        ddd          }|                    d           }||d<   g d|d<   t          ||           d S )NrC   r2   Europe/BrusselsrE   tzdatetime_tzTNTbool_with_none)r7   rO   r  r   )r   r*   ro   rP   dtis        r"   
test_basiczTestParquetPyArrow.test_basic  sb     mJ6GHHHnnT""=111R     r$   c                 |    |}t          j        ddd          |d<   t          |||ddg         dddgi	           d S )
NrC   r2   rm  rn  rp  r_   rd   r   r   )r7   rO   r   )r   r*   ro   rP   s       r"   test_basic_subset_columnsz,TestParquetPyArrow.test_basic_subset_columns  sf     M*aDUVVV=5)*"Xu$56		
 	
 	
 	
 	
 	
r$   c                    d}d }t                      rt          }|                    |          }t          |t                    sJ t          |          }t          |          }|                    d          }d |j        d<   t          j
        ||          5  t          j        ||           d d d            d S # 1 swxY w Y   d S )Nz.Mismatched null-like values nan and None foundr   F)deepru   r   )r   FutureWarningr   r   rb   r   r   r  ry   rz   assert_produces_warningr{   )	r   r*   ro   r   warn	buf_bytes
buf_streamresr   s	            r"   *test_to_bytes_without_path_or_buf_providedz=TestParquetPyArrow.test_to_bytes_without_path_or_buf_provided  s   >   	! D&&b&11	)U+++++Y''
:&&<<U<++-1)*'C888 	0 	0!'3///	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0s   B>>CCc                     t          j        t          j        d                              dd          t          d                                                    }|                     ||t          d           d S )N   r<   r2   aaar  zDuplicate column names found	r7   r8   rM   rN   reshaperK   r  r   r   r   r*   rP   s      r"   test_duplicate_columnsz)TestParquetPyArrow.test_duplicate_columns  sa    \")B--//155tE{{KKKPPRR!!"b*6TUUUUUr$   c                     t          j        dt          j        dd          i          }t          r|                     ||t
                     d S t          ||           d S )NrF   1 dayr2   rD   )r7   r8   timedelta_ranger	   r   NotImplementedErrorr   r  s      r"   test_timedeltaz!TestParquetPyArrow.test_timedelta  sc    \3 27A F F FGHH 	%..r27JKKKKKR$$$$$r$   c                 x    t          j        dg di          }|                     ||t          j                   d S )NrF   rF   r0   r\   )r7   r8   r   r   ArrowExceptionr  s      r"   test_unsupportedz#TestParquetPyArrow.test_unsupported  s=    \3.// 	**2r73IJJJJJr$   c                     t          j        ddt           j                  }t          j        |dg          }t
          r#|                     ||t          j                   d S t          ||           d S )Nr1   
   r@   fp16r   r   )
rM   rN   float16r7   r8   r   r   r   r  r   )r   r*   r   rP   s       r"   test_unsupported_float16z+TestParquetPyArrow.test_unsupported_float16  sq     yBbj111\tfX666 	%..r2w7MNNNNNR$$$$$r$   zqPyArrow does not cleanup of partial files dumps when unsupported dtypes are passed to_parquet function in windowsr   zfloat16 works on 15	path_typec                    t          j        ddt           j                  }t          j        |dg          }t          j                    5 } ||          }t          j        t          j	                  5  |
                    ||           d d d            n# 1 swxY w Y   t          j                            |          rJ 	 d d d            d S # 1 swxY w Y   d S )Nr1   r  r@   r  r  )r   r#   )rM   rN   r  r7   r8   rz   r   r   r   r  r   osr   isfile)r   r*   r  r   rP   path_strr   s          r"    test_unsupported_float16_cleanupz3TestParquetPyArrow.test_unsupported_float16_cleanup  sL    yBbj111\tfX666_ 	,(9X&&D)'*@AA 4 443334 4 4 4 4 4 4 4 4 4 4 4 4 4 4w~~d++++++		, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	,s6   *C6BCB	C!B	"$CCCc                 .   t          j                    }t          j        t          d                    |d<   t          j        g dt          j        g d                    |d<   t          j        g dg dd	
          |d<   t          ||           d S )NabcdefrF   )rX   r3   r3   rX   NrX   rW   r@   rG   )rF   rG   rS   rF   rS   rG   )rG   rS   rH   T)
categoriesorderedrS   )r7   r8   CategoricalrK   CategoricalDtyper   r  s      r"   test_categoricalz#TestParquetPyArrow.test_categorical  s    \^^.h003 .555%&;&;&;<<
 
 
3 .***PT
 
 
3 	R     r$   c                     t          j        d          } |j        di |}d|i}t          |||j         d||           d S )Ns3fs
filesystem/pyarrow.parquetr   r   r   r)   )r'   r   S3FileSystemr   r  )r   r9   s3_public_bucketr*   s3sor  s3kws           r"   test_s3_roundtrip_explicit_fsz0TestParquetPyArrow.test_s3_roundtrip_explicit_fs$  sp    "6**T&&&&B$);;;	
 	
 	
 	
 	
 	
r$   c                 H    d|i}t          ||d|j         d||           d S )Nstorage_optionss3://r  r  r   r  )r   r9   r  r*   r  s        r"   test_s3_roundtripz$TestParquetPyArrow.test_s3_roundtrip1  sM     "4(@).@@@	
 	
 	
 	
 	
 	
r$   partition_colr5   c                 V   t          j        d           |                                }|rX|                    t                              |t          j                            }d}||                             |          ||<   t          |||d|j	         dd|i|d |ddd	           d S )
Nr  categoryr  z/parquet_dirr  )r   rs   r  Tr0   )r   r   r   r   rw   r|   )
r'   r   r  rl   dictfromkeysrM   int32r   r  )r   r9   r  r*   r  r  expected_dfpartition_col_types           r"   test_s3_roundtrip_for_dirz,TestParquetPyArrow.test_s3_roundtrip_for_dir=  s     	F###nn&&  	%,,T]]="(-S-STTK!+)4])C)J)J"* *K& 	 <).<<<*D1"/##' 
 	
 	
 	
 	
 	
 	
r$   c                     t          j        d           t                      }|                    |           t	          |          }t          j        ||           d S )Nr   )r'   r   r   r   r   rz   r{   )r   r9   bufferdf_from_bufs       r"   test_read_file_like_obj_supportz2TestParquetPyArrow.test_read_file_like_obj_supportd  sV    I&&&V$$$"6**
i55555r$   c                    t          j        d           |                    dd           |                    dd           t          j        t          d          5  t          d           d d d            n# 1 swxY w Y   t          j        t          d          5  |                    d           d d d            d S # 1 swxY w Y   d S )Nr   HOMETestingUserUSERPROFILEz.*TestingUser.*r   z~/file.parquet)r'   r   setenvr   OSErrorr   r   )r   r9   monkeypatchs      r"   test_expand_userz#TestParquetPyArrow.test_expand_userk  sH   I&&&6=111=-888]7*<=== 	+ 	+)***	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+]7*<=== 	3 	3  !1222	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3s$   A88A<?A<CCCc                     ddg}|}|                     ||d            t          ||           t          |          j        |j        k    sJ d S )Nrh   rd   r   rs   r   r   r   shape)r   r   r*   ro   r   rP   s         r"   test_partition_cols_supportedz0TestParquetPyArrow.test_partition_cols_supportedt  s\     %
h~4PPPh777H%%+rx777777r$   c                     d}|g}|}|                     ||d            t          ||           t          |          j        |j        k    sJ d S )Nrh   r  r  )r   r   r*   ro   r   partition_cols_listrP   s          r"   test_partition_cols_stringz-TestParquetPyArrow.test_partition_cols_string|  sb    -.
h~4PPPh(;<<<H%%+rx777777r$   c                     | S r   r)   )xs    r"   <lambda>zTestParquetPyArrow.<lambda>  s    Q r$   r_   zpathlib.Path)idsc                     d}|g}|} ||          }|                     ||           t          |          j        |j        k    sJ d S )Nr6   )r   )r   r   r  )	r   r   r*   r9   r  r   r  rP   r   s	            r"   test_partition_cols_pathlibz.TestParquetPyArrow.test_partition_cols_pathlib  sa     -.y""
d+>???D!!'28333333r$   c                 R    t          j        g g           }t          ||           d S )N)r  r   r   r  s      r"   test_empty_dataframez'TestParquetPyArrow.test_empty_dataframe  s,    \B///R     r$   c                     dd l }t          j        dddgi          } |j         |j        d |j                              g          }|                    t                    }t          ||d|i|           d S )Nr   r  r0   )typer   r'  )	r   r7   r8   r   fieldbool_rl   rh   r   )r   r*   r   rP   r   out_dfs         r"   test_write_with_schemaz)TestParquetPyArrow.test_write_with_schema  s    \3A-((s!I!I!I JKK4Rx.@6RRRRRRr$   c                 z   t          j        d           t          j        t          j        g dd          t          j        g dd          t          j        g dd          d          }t          ||           t          j        d	t          j        g d
d          i          }t          ||           d S )Nr   r/   rY  r@   UInt32rT   r_   rF   rG   rS   rF   rP  )r'   r   r7   r8   r   r   r  s      r"    test_additional_extension_arraysz3TestParquetPyArrow.test_additional_extension_arrays  s     	I&&&\Yyyy888Yyyy999Y///x@@@ 
 
 	R   \3	/// I I IJKKR     r$   c           
      4   t          j        d           t          j        dt          j        g dd          i          }t          j        d|          5  t          |||                    d| d          	           d d d            d S # 1 swxY w Y   d S )
Nr   rF   rT   string[pyarrow]r@   string_storagezstring[]r   )r'   r   r7   r8   r   r   r   rl   )r   r*   r  rP   s       r"    test_pyarrow_backed_string_arrayz3TestParquetPyArrow.test_pyarrow_backed_string_array  s    I&&&\3	*:*:*:BS T T TUVV/@@ 	V 	VRbii8S.8S8S8S.T.TUUUU	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	Vs   *BBBc                 J   t          j        d           t          j        t          j                            g d          t          j        ddd          t          j                            t          j        ddd                    d          }t          ||           d S )	Nr   ))r   r0   )r0   r1   )r2   r<   z
2012-01-01r2   D)rE   r  r<   )rS   rH   rI   )
r'   r   r7   r8   IntervalIndexr  period_rangefrom_breaksrO   r   r  s      r"   test_additional_extension_typesz2TestParquetPyArrow.test_additional_extension_types  s     	I&&&\%112J2J2JKK_\13GGG%11M,DDD 	 	
 	
 	R     r$   c                     t           sd}nd}t          j        dt          j        ddd          i          }t	          ||d|i	           d S )
Nz2.6z2.0rF   z
2017-01-011nr  r  rE   versionr   )r   r7   r8   rO   r   )r   r*   verrP   s       r"   test_timestamp_nanosecondsz-TestParquetPyArrow.test_timestamp_nanoseconds  s]     # 	CCC\3lr R R RSTTRy#.>??????r$   c                    t           sR|j        t          j        j        k    r8|j                            t          j        	                    d                     d|gz  }t          j        |d|i          }t          ||d           d S )Nzitemporary skip this test until it is properly resolved: https://github.com/pandas-dev/pandas/issues/37286r      index_as_colr  r   F)rx   )r   tzinfori   timezoneutcr  r  r'   r   r  r7   r8   r   )r   r!   r*   rq   idxrP   s         r"   test_timezone_aware_indexz,TestParquetPyArrow.test_timezone_aware_index  s    #		(/83D3HHHL##!!H "     +,,\>3*?@@@ 	RU333333r$   c           	      d   t          j        d           t          j        dt	          t          dd                    i          }t          j                    5 }|                    ||           t          ||dgd          }d d d            n# 1 swxY w Y   t          |          dk    sJ d S )	Nr   rF   r   r2   rF   r   r   F)r   use_legacy_datasetr0   )r'   r   r7   r8   rK   rL   rz   r   r   r   r   )r   r*   rP   r   r   s        r"   test_filter_row_groupsz)TestParquetPyArrow.test_filter_row_groups  s    I&&&\3U1a[[ 1 1233_ 	$MM$###!b>"2u  F	 	 	 	 	 	 	 	 	 	 	 	 	 	 	
 6{{as   +BBBc                    t          j        t          j                            d                              d          g d          }t          j                    5 }|                    ||           t          ||          }d d d            n# 1 swxY w Y   |r-t          |j        t           j        j        j                  sJ d S t          |j        t           j        j        j                  sJ d S )Nr1   )r  r2   )r5   r6   Cr  )r7   r8   rM   r  r  r  rz   r   r   r   r   _mgrcore	internalsArrayManagerBlockManager)r   r*   using_array_managerrP   r   r   s         r"   test_read_parquet_managerz,TestParquetPyArrow.test_read_parquet_manager  s   \I!!!$$44W==
 
 
 _ 	,$MM$###!$++F	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	,  	Kfk27+<+IJJJJJJJfk27+<+IJJJJJJJs   'BBBc                 &   dd l }|}t          j        ddd          }|                    d           }||d<   g d|d<   |j                            |          }|                    t          j        	          }t          r||d
         	                    d          |d
<   |d         	                    d          |d<   |d         	                    t          j         |j
        dd                              |d<   t          ||ddi|           d S )Nr   rC   r2   rm  rn  rp  rq  rr  )types_mapperri   ztimestamp[us][pyarrow]rj   us)unitro  rX  r   r#   r   r   )r   r7   rO   r  Tablefrom_pandas	to_pandas
ArrowDtyper   rl   	timestampr   )r   r*   ro   r   rP   rs  pa_tabler   s           r"   &test_read_dtype_backend_pyarrow_configz9TestParquetPyArrow.test_read_dtype_backend_pyarrow_config  sB    mJ6GHHHnnT""=111=,,R00%%2=%AA 	#+J#7#>#>?W#X#XHZ ,45H,I,P,P(- -H() '/}&=&D&D/g/T>OPPPQQ' 'H]# 	()4		
 	
 	
 	
 	
 	
r$   c                 J   t          j        dddgit          j        ddgd          d	          }|                                }d
d l}t          |j                  t          d          k    r|j                            d          |_        t          ||ddi|           d S )NrF   r0   r1   r2   r<   testr  zint64[pyarrow])r  rA   r   z11.0.0rX  r   r  )
r7   r8   Indexr  r   r   r   r  rl   r   )r   r*   rP   r   r   s        r"   ,test_read_dtype_backend_pyarrow_config_indexz?TestParquetPyArrow.test_read_dtype_backend_pyarrow_config_index"  s    \1a&M1a&v!>!>!>FV
 
 
 77997&'''(*;*;;;%^223CDDHN()4		
 	
 	
 	
 	
 	
r$   c           	         t          j        t          d          t          t          dd                    d          }ddg|_        t          ||           ddg|_        t          j        t          d	          5  t          ||           d d d            n# 1 swxY w Y   t          j	        d
dddd          t          j	        d
dddd          g|_        t          ||           d S )Nr;   r0   r<   r   r   rU   rV   z|S3r     )
r7   r8   rK   rL   r   r   r'   r   r  ri   r  s      r"   test_columns_dtypes_not_invalidz2TestParquetPyArrow.test_columns_dtypes_not_invalid2  s$   \T%[[eAqkk9J9JKKLL V
R    f%
].e<<< 	% 	%R$$$	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% dAq!Q//dAq!Q//

 	R     s   =BB!Bc                 |    t          j        t          j        g dd                    }t          ||           d S )Nr  custom namer  r+  r7   r8   r  r   r  s      r"   test_empty_columnsz%TestParquetPyArrow.test_empty_columnsF  s=    \} M M MNNNR     r$   c                     |dz  }t          j        ddgi          }ddi|_        |                    ||           t	          ||          }|j        |j        k    sJ d S )Nztest_df_metadata.pr0   r   test_attributer   )r7   r8   attrsr   r   )r   r   r*   r   rP   new_dfs         r"   test_df_attrs_persistencez,TestParquetPyArrow.test_df_attrs_persistenceK  st    ..\A3x((($a(
d2&&&d2...|rx''''''r$   c                 ~   |dz  }t          j        dddgiddg          }|                    |d           t          j        d	d
          5  t	          |d          }d d d            n# 1 swxY w Y   t          j        dddgidt          j        ddgd                    }t          j        ||           d S )Nztest_string_inference.prF   r  yrG   )r   r  r   r   future.infer_stringTstring[pyarrow_numpy]r@   )r   rA   r  )r7   r8   r   r   r   r  rz   r{   r   r   r*   r   rP   r   r   s          r"   test_string_inferencez(TestParquetPyArrow.test_string_inferenceS  s   33\c3Z0c
CCC
d9---4d;; 	: 	:!$y999F	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	:<Sz")(C:-DEEE
 
 

 	fh/////s   A,,A03A0znot supported before 11.0c           	      ^   dd l }|dz  }t          j        dt          d          gid          }|                    ||                    d|                    d          fg                     t          |          }t          j        dd	gid
          }t          j	        ||           d S )Nr   z	decimal.prF   z123.00r  r@   r  )r   123zstring[python])
r   r7   r8   r   r   r   
decimal128r   rz   r{   r,  s          r"   test_roundtrip_decimalz)TestParquetPyArrow.test_roundtrip_decimala  s     	+%\3!2!2 34<MNNN
d299sBMM!4D4D.E-F#G#GHHHd##<ug6FGGG
fh/////r$   c                    dd l }dd lm} |dz  }|                    d|                    g d|                                          i          }|                    ||           t          j        dd          5  t          |          }d d d            n# 1 swxY w Y   t          j
        dg didt          j        dgd          	          }t          j        ||           d S )
Nr   zlarge_string.prF   )NrG   rS   r*  Tr+  r@   )r   rA   r   )r   r   r   r_  r   large_stringr`  r7   r   r   r8   r  rz   r{   )r   r   r*   r   r   r_  r   r   s           r"   #test_infer_string_large_string_typez6TestParquetPyArrow.test_infer_string_large_string_typem  sB   $$$$$$**#rxx(8(8(8"//:K:KLLMNN
ud###4d;; 	( 	(!$''F	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	(<'''()HcU*ABBB
 
 

 	fh/////s   :BBBN)1r   r   r   rt  rv  r  r  r  r  r  r'   r   r  r   ri  r   rh  strpathlibPathr  r  r   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r	  r  r  r  r!  r'  r-  r
   r1  r4  r)   r$   r"   rk  rk    sj       	! 	! 	!
 
 
0 0 0$V V V
% % %K K K% % % [?    [009NOO[[3*=>>, , ?> PO ,! ! !$ [

 

 

 [	
 	
 	
 [[E	
 
 
  
>6 6 63 3 38 8 88 8 8 [c;;'h-G   	4 	4 	4! ! !
S S S! ! ! V V V! ! ! 	@ 	@ 	@4 4 42	  	  	 K K K
 
 
<
 
 
 ! ! !(! ! !
( ( (0 0 0 [,5PQQ	0 	0 RQ	00 0 0 0 0r$   rk  c                      e Zd Zd Zd Zd Zd Zd Zd Zd Z	e
j        j        d             Zd	 Zd
 Zd Zd Ze
j                             e            d          d             Ze
j                             e            d          d             Zd Zd Zd Zd Zd Zd Zd Ze
j                             e            d          d             ZdS )TestParquetFastParquetc                     |}t          j        ddd          }|                    d           }||d<   t          j        dd          |d<   t	          ||           d S )	NrC   r2   z
US/Easternrn  rp  r  rD   	timedelta)r7   rO   r  r  r   )r   r-   ro   rP   rs  s        r"   rt  z!TestParquetFastParquet.test_basic  sg    mJlCCCnnT""=,Wa@@@;R     r$   c           	         t          j        t          d          t          t          dd                    d          }t          }d}ddg|_        |                     ||||           ddg|_        |                     ||||           t          j        d	dddd          t          j        d	dddd          g|_        |                     ||||           d S )
Nr;   r0   r<   r   r:  r   rU   rV   r  )r7   r8   rK   rL   r;  r   r   ri   )r   r-   rP   errr   s        r"   test_columns_dtypes_invalidz2TestParquetFastParquet.test_columns_dtypes_invalid  s    \T%[[eAqkk9J9JKKLL, V
!!"b#s333 f%
!!"b#s333 dAq!Q//dAq!Q//

 	!!"b#s33333r$   c                     t          j        t          j        d                              dd          t          d                                                    }d}|                     ||t          |           d S )Nr  r<   r2   r  r  z9Cannot create parquet dataset with duplicate column namesr  r   r-   rP   r   s       r"   r  z-TestParquetFastParquet.test_duplicate_columns  se    \")B--//155tE{{KKKPPRRI!!"b*c:::::r$   c                     t          j        dg di          }t          j        ddt          j        dgid          }t	          |||d           d S )	NrF   )TNFrS  g        r  r@   F)r   rx   )r7   r8   rM   rk   r   r   r-   rP   r   s       r"   test_bool_with_nonez*TestParquetFastParquet.test_bool_with_none  s^    \3 3 3 3455<sBFC&8 9KKK 	R(FFFFFFr$   c                    t          j        dt          j        ddd          i          }|                     ||t          d            t          j        dg di          }d}|                     ||t          |           d S )NrF   2013Mr2   r  r  z"Can't infer object conversion type)r7   r8   r  r   r   r@  s       r"   r  z'TestParquetFastParquet.test_unsupported  s    \3S! L L LMNN!!"b*d;;; \3.//2!!"b*c:::::r$   c                     t          j        dt          j        t          d                    i          }t	          ||           d S )NrF   r;   )r7   r8   r  rK   r   )r   r-   rP   s      r"   r  z'TestParquetFastParquet.test_categorical  s<    \3tE{{ ; ;<==R     r$   c                 D   dt          t          dd                    i}t          j        |          }t	          j                    5 }|                    ||d d           t          ||dg          }d d d            n# 1 swxY w Y   t          |          dk    sJ d S )NrF   r   r2   r0   )rs   row_group_offsetsr  )r   )	rK   rL   r7   r8   rz   r   r   r   r   )r   r-   rH   rP   r   r   s         r"   r   z-TestParquetFastParquet.test_filter_row_groups  s    $uQ{{##$\!___ 	F$MM$MJJJ!$^4DEEEF	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 6{{as   -B  BBc           	      J    t          ||d|j         dd|id |d           d S )Nr  z/fastparquet.parquetr  )rs   r  r  r  )r   r9   r  r-   r  s        r"   r  z(TestParquetFastParquet.test_s3_roundtrip  sP     	D).DDD*D1)-$GG	
 	
 	
 	
 	
 	
r$   c                     ddg}|}|                     |d|d            t          j                            |          sJ dd l} |j        t          |          d          j        }t          |          dk    sJ d S )Nrh   rd   r   r#   r   rs   r   Fr1   	r   r  r   existsr   ParquetFiler5  catsr   r   r   r-   ro   r   rP   r   actual_partition_colss           r"   r  z4TestParquetFastParquet.test_partition_cols_supported  s     %
 )	 	 	
 	
 	
 w~~h''''' 7 7Hu M M R())Q......r$   c                     d}|}|                     |d|d            t          j                            |          sJ dd l} |j        t          |          d          j        }t          |          dk    sJ d S )Nrh   r   rL  r   Fr0   rM  rQ  s           r"   r  z1TestParquetFastParquet.test_partition_cols_string  s    
 )	 	 	
 	
 	
 w~~h''''' 7 7Hu M M R())Q......r$   c                     ddg}|}|                     |dd |           t          j                            |          sJ dd l} |j        t          |          d          j        }t          |          dk    sJ d S )Nrh   rd   r   )r#   rs   partition_onr   Fr1   rM  rQ  s           r"   test_partition_on_supportedz2TestParquetFastParquet.test_partition_on_supported  s     %
 '	 	 	
 	
 	
 w~~h''''' 7 7Hu M M R())Q......r$   c                     ddg}|}d}t          j        t          |          5  |                    |dd ||           d d d            d S # 1 swxY w Y   d S )Nrh   rd   zYCannot use both partition_on and partition_cols. Use partition_cols for partitioning datar   r   )r#   rs   rU  r   )r'   r   r   r   )r   r   r-   ro   r   rP   r   s          r"   3test_error_on_using_partition_cols_and_partition_onzJTestParquetFastParquet.test_error_on_using_partition_cols_and_partition_on  s     !%  	 ]:S111 	 	MM$ +-    	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	s   AAAzfastparquet writes into Indexr   c                 x    t          j                    }|                                }t          |||           d S )Nr  )r7   r8   r  r   rB  s       r"   r  z+TestParquetFastParquet.test_empty_dataframe  s7     \^^7799R(333333r$   c                     d|gz  }t          j        |d|i          }|                                }d|j        _        t          |||           d S )Nr  r  r  r  r  )r7   r8   r  r  r  r   )r   r-   rq   r  rP   r   s         r"   r  z0TestParquetFastParquet.test_timezone_aware_index%  s[    +,,\>3*?@@@7799%R(333333r$   c                 .   t          j        dddgi          }t          j                    5 }|                    |           t          j        t          d          5  t          j        t                    5  t          |dd           d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   t          j        t          d          5  t          |dd	
           d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )NrF   r0   r1   z!not supported for the fastparquetr   r   T)r#   use_nullable_dtypesr   rW  )r7   r8   rz   r   r   r'   r   r   rz  ry  r   )r   r-   rP   r   s       r"   &test_use_nullable_dtypes_not_supportedz=TestParquetFastParquet.test_use_nullable_dtypes_not_supported/  sI   \3A-((_ 	R$MM$z1TUUU W W/>> W W mQUVVVVW W W W W W W W W W W W W W WW W W W W W W W W W W W W W W z1TUUU R RT-yQQQQR R R R R R R R R R R R R R R	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	Rs}   1D
B-7B
B-BB-BB-!D
-B1	1D
4B1	5D
C2&D
2C6	6D
9C6	:D

DDc                    t          j        d          5 }t          j        |                              d           t          j        t          d          5  t          |d           d d d            n# 1 swxY w Y   t          j        |          	                    d           d d d            d S # 1 swxY w Y   d S )	Ntest.parquets   breakit r   r   r   F)
missing_ok)
rz   r   r6  r7  write_bytesr'   r   	Exceptionr   unlink)r   r   s     r"   $test_close_file_handle_on_read_errorz;TestParquetFastParquet.test_close_file_handle_on_read_error:  s1   _^,, 	8L**:666y333 9 9T-88889 9 9 9 9 9 9 9 9 9 9 9 9 9 9 L%%%777	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8s6   AB6A6*B66A:	:B6=A:	>+B66B:=B:c                    t          j        ddgddgd          }t          j        d          5 }t	          |                                d          5 }|                    |           d d d            n# 1 swxY w Y   t          ||          }d d d            n# 1 swxY w Y   t          j        ||           d S )Nr   r0   r4   r#  r_  wbr   )	r7   r8   rz   r   r   encoder   r   r{   )r   r#   rP   r   rJ   r   s         r"   test_bytes_file_namez+TestParquetFastParquet.test_bytes_file_nameB  s1   \aV1a&99:::_^,, 	7dkkmmT** !aa   ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! "$v666F		7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7
 	fb)))))s5   #BA6*B6A:	:B=A:	>BB"%B"c                    t          j        d           t          j        ddgddgd          }t	          j                    5 }t          j        t          d          5  |                    |dd	           d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   t	          j                    5 }t          j
        |                              d
           t          j        t          d          5  t          |dd	           d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nr   r   r0   r4   r#  zfilesystem is not implementedr   r3   r#   r  rU   )r'   r   r7   r8   rz   r   r   r  r   r6  r7  rb  r   r   rP   r   s      r"   test_filesystem_notimplementedz5TestParquetFastParquet.test_filesystem_notimplementedL  s~   M***\aV1a&99:::_ 	L$#+J   L L d=UKKKL L L L L L L L L L L L L L L	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	L _ 	K$L**6222#+J   K K T-EJJJJK K K K K K K K K K K K K K K	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	Km   B B9BB			BB		BB #B :AD4=DD4D 	 D4#D 	$D44D8;D8c                    t          j        d           t          j        ddgddgd          }t	          j                    5 }t          j        t          d          5  |                    |dd	           d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   t	          j                    5 }t          j
        |                              d
           t          j        t          d          5  t          |dd	           d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nr   r   r0   r4   r#  z1filesystem must be a pyarrow or fsspec FileSystemr   r3   rk  rU   )r'   r   r7   r8   rz   r   r   r   r   r6  r7  rb  r   rl  s      r"   test_invalid_filesystemz.TestParquetFastParquet.test_invalid_filesystem\  s~   I&&&\aV1a&99:::_ 	H$"U   H H d9GGGH H H H H H H H H H H H H H H	H 	H 	H 	H 	H 	H 	H 	H 	H 	H 	H 	H 	H 	H 	H _ 	G$L**6222"U   G G T)FFFFG G G G G G G G G G G G G G G	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	Grn  c           	         t          j        d          }t          j        ddgddgd          }t	          j                    5 }t          j        t          d          5  |                    |d|	                                d	d
i           d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   t	          j                    5 }t          j        |                              d           t          j        t          d          5  t          |d|	                                d	d
i           d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nz
pyarrow.fsr   r0   r4   r#  z8storage_options not supported with a pyarrow FileSystem.r   r   r3   rX   )r#   r  r  rU   )r'   r   r7   r8   rz   r   r   r  r   LocalFileSystemr6  r7  rb  r   )r   pa_fsrP   r   s       r"   .test_unsupported_pa_filesystem_storage_optionszETestParquetFastParquet.test_unsupported_pa_filesystem_storage_optionsl  su   #L11\aV1a&99:::_ 
	$#P   	 	 $$4466%*EN	    		 	 	 	 	 	 	 	 	 	 	 	 	 	 	
	 
	 
	 
	 
	 
	 
	 
	 
	 
	 
	 
	 
	 
	 
	 _ 	$L**6222#P   	 	 $$4466%*EN	   		 	 	 	 	 	 	 	 	 	 	 	 	 	 		 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	sm   B1 .BB1B	B1!B	"B11B58B5AE(E:EE
	
EE
	EE"%E"c           	      x   d}t          j        dt          t          dd                    i          }t	          j        d          5 }|                    |           t          j        t          |          5  t          |d           d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )	NzPdtype_backend numpy is invalid, only 'numpy_nullable' and 'pyarrow' are allowed.rd   r0   r<   ztmp.parquetr   numpy)rX  )r7   r8   rK   rL   rz   r   r   r'   r   r   r   )r   r#   r   rP   r   s        r"   test_invalid_dtype_backendz1TestParquetFastParquet.test_invalid_dtype_backend  s2   % 	 \5$uQ{{"3"3455_]++ 	:tMM$z555 : :T9999: : : : : : : : : : : : : : :	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	:s6   1B/9BB/B	B/B	B//B36B3c                     t          j        t          j        g dd                    }t          j        t          j        g dd                    }t          |||           d S )Nr  r  r  r+  r  r   rB  s       r"   r!  z)TestParquetFastParquet.test_empty_columns  si     \} M M MNNN<bh]&S&S&STTTR(333333r$   N)r   r   r   rt  r>  r  rC  r  r  r   r'   r   r   r  r  r  rV  rX  ri  r   r  r  r]  re  ri  rm  rp  rt  rw  r!  r)   r$   r"   r9  r9    s       ! ! !4 4 4*; ; ;G G G	; 	; 	;! ! !      [
 
 
/ / / / / / / / /   & [++--6UVV4 4 WV4 [++--6UVV4 4 WV4	R 	R 	R8 8 8* * *K K K G G G   8	: 	: 	: [++--6UVV4 4 WV4 4 4r$   r9  )	NNNNNTFTr1   )I__doc__ri   decimalr   r   r   r  r6  rv  rM   r'   pandas._configr   r   pandas.compatr   pandas.compat.pyarrowr   r	   r
   r   r   pandasr7   pandas._testing_testingrz   pandas.util.versionr   pandas.io.parquetr   r   r   r   r   r   r&   r   r   r,   r   filterwarnings
pytestmarkfixturer   ri  r#   r*   r-   r9   rQ   ro   nowr  r  minmaxstrptimerq   r   r   r   r   r   r   r   r   r   r   r   r   rk  r9  r)   r$   r"   <module>r     s                  				             
 . - - - - -                        ' ' ' ' ' '             NNNMM   MMM    [''8 
 +$$%%S4G)H)HG)SM %  	
 	
 	
 	+$$!!*D %  	
 	
 	
  " # "       6 6 6   "   . h/344h/344h/344""#=?TUU""#=?TUU""#=?TUU""#=?TUU
 
 
 
 
 	? ? ? ?D= = =.2 2 2$ $ $$ $ $$ $ $F F F((# (# (#V6 6 6
6 
6 
6- - - - - - - -0k
 k
 k
 k
 k
 k
 k
 k
\	K0 K0 K0 K0 K0 K0 K0 K0tK4 K4 K4 K4 K4T K4 K4 K4 K4 K4s$   A% %A/.A/3A: :BB