
    'e                        d dl Z d dlmZ d dlmZ d dlZd dlmZ d dlm	Z	 d dl
Z
d dlZd dlZd dlZd dlZd dlZd dlmZ d dlZd dlmZmZ i ad Zd	 Zej        d
ej        dej        dej        dej         dej!        dej"        dej#        dej$        dej%        dej&        dddej'        dej(        diZ)d Z*d Z+d Z,d Z-d Z.d Z/d Z0d  Z1d! Z2d" Z3d# Z4d$ Z5d% Z6d& Z7dEd'Z8	 	 dFd*Z9d+ Z:dGd,Z;d- Z<	 dHd/Z=d0 ed1         ed2         z   ed3         z   d4d
gz   D             Z>dEd5Z?d6 Z@d7 ZAdEd8ZB	 dEd9ZCd: ZDd; ZEdd<d<ej'        ej(        ej'        ej         ej&        ejF        d=	ZGd> ZHd? ZId@ ZJdA ZKdB ZLdC ZMdD ZNdS )I    N)Sequence)futures)deepcopy)zip_longest)sctypes)_pandas_api	frombytesc            	      "   t           st                               i t          j        j        dt          j        j        dt          j        j        dt          j        j        dt          j        j        dt          j        j	        dt          j        j
        dt          j        j        dt          j        j        d	t          j        j        d
t          j        j        dt          j        j        dt          j        j        dt          j        j        dt          j        j        dt          j        j        dt          j        j        dt          j        j        dt          j        j        dt          j        j        di           t           S )Nemptyboolint8int16int32int64uint8uint16uint32uint64float16float32float64datetimebytesunicode)_logical_type_mapupdatepalibType_NA	Type_BOOL	Type_INT8
Type_INT16
Type_INT32
Type_INT64
Type_UINT8Type_UINT16Type_UINT32Type_UINT64Type_HALF_FLOAT
Type_FLOATType_DOUBLEType_DATE32Type_DATE64Type_TIME32Type_TIME64Type_BINARYType_FIXED_SIZE_BINARYType_STRING     5  /croot/pyarrow_1707330824290/_h_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold/lib/python3.11/site-packages/pyarrow/pandas_compat.pyget_logical_type_mapr7   +   sw        "
FNG"
Ff"
 Ff"
 Fw	"

 Fw"
 Fw"
 Fw"
 F"
 F"
 F"
 F"I"
 Fy"
 F	"
 F"
 F"
  F!"
" F#"
$ FF)7F	)"
 "
 	 	 	, r5   c                    t                      }	 || j                 S # t          $ r t          | t          j        j                  rY dS t          | t          j        j                  r)d                    t          | j
                            cY S t          | t          j        j                  r| j        dndcY S t          | t          j        j                  rY dS Y dS w xY w)Ncategoricalzlist[{}]
datetimetzdatetimedecimalobject)r7   idKeyError
isinstancer   r   DictionaryTypeListTypeformatget_logical_type
value_typeTimestampTypetzDecimal128Type)
arrow_typelogical_type_maps     r6   rD   rD   H   s    +--
.. 	 	 	j"&"788 	 ==
BFO44 	$$%5j6K%L%LMMMMM
BF$899 	#-=#<<<*LLL
BF$9:: 	99xx	s"    )C#	AC#*C#=C#"C#r   r   r   r   r   r   r   r   r   r   r   zdatetime64[D]r   stringr   c                     	 t           | j        j                 S # t          $ rU t	          | j        d          rY dS t          | j                  dk    rY dS t          j        |           }|dk    rY dS |cY S w xY w)NrG   r:   datetime64[ns]rK   r   )_numpy_logical_type_mapdtypetyper?   hasattrstrr   infer_dtype)pandas_collectionresults     r6   get_logical_type_from_numpyrV   k   s    &'8'>'CDD 	 	 	$*D11 	 << &''+;;;###():;;X99	s     A8A8A83A87A8c                 h   | j         }t          |          dk    rKt          | d|           }|J t          |j                  |j        d}t          |j        j                   }nJt          |d          r)dt          j	        
                    |j                  i}d}nd }t          |          }||fS )Ncategorycat)num_categoriesorderedrG   timezonerM   )rO   rR   getattrlen
categoriesr[   codesrQ   r   r   tzinfo_to_stringrG   )columnrO   catsmetadataphysical_dtypes        r6   get_extension_dtype_inforf   z   s    LE
5zzZvuf--!$/22|
 
 TZ-..			 $ 7 7 A AB)U8##r5   c                    t          |          }t          |           \  }}|dk    r|j        |j        d}d}|Jt	          |t
                    s5t          d                    |t          |          j	                            |9t	          |t
                    s$J t          t          |                                ||dn||||dS )a  Construct the metadata for a given column

    Parameters
    ----------
    column : pandas.Series or pandas.Index
    name : str
    arrow_type : pyarrow.DataType
    field_name : str
        Equivalent to `name` when `column` is a `Series`, otherwise if `column`
        is a pandas Index then `field_name` will not be the same as `name`.
        This is the name of the field in the arrow Table's schema.

    Returns
    -------
    dict
    r<   )	precisionscaler=   Nz6Column name must be a string. Got column {} of type {}Nonename
field_namepandas_type
numpy_typerd   )
rD   rf   rh   ri   r@   rR   	TypeErrorrC   rP   __name__)rb   rl   rI   rm   logical_typestring_dtypeextra_metadatas          r6   get_column_metadataru      s    " $J//L#;F#C#C L.y  #-%
 
  
4 5 5DKKd4jj) 
 
 	
 J!<!<D   * 2ff
#""  r5   c           	      \   t          d |D                       }t          |          }|d||z
           }	|||z
  d         }
g }t          | ||	          D ].\  }}}t          ||||          }|                    |           /g }|durSg }t          ||
|          D ]\  }}}t	          |t
                    r|j        4t	          |j        t                    s|                    |j                   t          |t          |j                  ||          }|                    |           t          |          dk    r t          j
        d| dt          d	           g }t          |j        d
|j        g          }t          |j        d|j        j        g          }t          ||          D ]*\  }}t          ||          }|                    |           +ng x}x}}dt          j        ||||z   dt"          j        dt&          j        d                              d          iS )a  Returns a dictionary containing enough metadata to reconstruct a pandas
    DataFrame as an Arrow Table, including index columns.

    Parameters
    ----------
    columns_to_convert : list[pd.Series]
    df : pandas.DataFrame
    index_levels : List[pd.Index]
    index_descriptors : List[Dict]
    preserve_index : bool
    types : List[pyarrow.DataType]

    Returns
    -------
    dict
    c                 <    g | ]}t          |t                    |S r4   )r@   dict).0descrs     r6   
<listcomp>z&construct_metadata.<locals>.<listcomp>   s=     'G 'G 'G.8.E.E'Gu 'G 'G 'Gr5   N)rl   rI   rm   Fr   z&The DataFrame has non-str index name `z@` which will be converted to string and not roundtrip correctly.   
stacklevellevelsnamess   pandaspyarrow)libraryversion)index_columnscolumn_indexescolumnscreatorpandas_versionutf8)r^   zipru   appendr@   rx   rl   rR   _column_name_to_stringswarningswarnUserWarningr]   r   _get_simple_index_descriptorjsondumpsr   __version__r   r   encode)columns_to_convertdfcolumn_namesindex_levelsindex_descriptorspreserve_indextypesnum_serialized_index_levelsntypesdf_typesindex_typescolumn_metadatacolsanitized_namerI   rd   index_column_metadatanon_str_index_nameslevel
descriptorr   r   r   rl   s                           r6   construct_metadatar      s   $ #& 'G 'G:K 'G 'G 'G #H #H ZZF:f:::;H!<<==>KO+./A/;X,G ,G ) )'^Z&s2<2@B B B 	x((((U"" -0{1B.D .D 	3 	3)E:z*d++  z%jS.I.I%#**5:666*,UZ88%%	  H "((2222"##a''M09L 0 0 0 	+ + + + X
|<<
Gbjo->??vu-- 	, 	,KE43E4@@H!!(++++	, FHGG1N 	4:.,&)>>$>  *1	
 	
 	 	 6&>> r5   c                     t          |           \  }}t          |           }d|v rt          j        dt          d           |dk    r|rJ ddi}|||||dS )	NmixedzlThe DataFrame has column names of mixed type. They will be converted to strings and not roundtrip correctly.r|   r}   r   encodingUTF-8rk   )rf   rV   r   r   r   )r   rl   rs   rt   rn   s        r6   r   r     s    #;E#B#B L.-e44K+@A	' 	' 	' 	' i!!!!$g."""  r5   c                 z   t          | t                    r| S t          | t                    r|                     d          S t          | t                    r/t          t	          t          t          |                               S t          | t                    rt          d          | dS t          |           S )a!  Convert a column name (or level) to either a string or a recursive
    collection of strings.

    Parameters
    ----------
    name : str or tuple

    Returns
    -------
    value : str or tuple

    Examples
    --------
    >>> name = 'foo'
    >>> _column_name_to_strings(name)
    'foo'
    >>> name = ('foo', 'bar')
    >>> _column_name_to_strings(name)
    "('foo', 'bar')"
    >>> import pandas as pd
    >>> name = (1, pd.Timestamp('2017-02-01 00:00:00'))
    >>> _column_name_to_strings(name)
    "('1', '2017-02-01 00:00:00')"
    r   z%Unsupported type for MultiIndex levelN)	r@   rR   r   decodetuplemapr   r   rp   rl   s    r6   r   r   $  s    2 $ 
	D%	 	  {{6"""	D%	 	  54d;;<<===	D(	#	# ?@@@	tt99r5   c                 t    | j         | j         |vrt          | j                   S d                    |          S )zReturn the name of an index level or a default name if `index.name` is
    None or is already a column name.

    Parameters
    ----------
    index : pandas.Index
    i : int

    Returns
    -------
    name : str
    Nz__index_level_{:d}__)rl   r   rC   )indexir   s      r6   _index_level_namer   K  s<     z%*L"@"@&uz222%,,Q///r5   c                    t          | ||          }| j        j        s4t          d                    t          | j                                      |t          | ||          S g }|durt          | j                  ng }g }g }|D ]}| |         }	t          |          }t          j        |	          r"t          d                    |                    |                    |	           |                    d            |                    |           g }
g }t          |          D ]\  }}t          |||          }t!          |t          j        j                  r|t'          |          }nA|                    |           |                    d            |}|                    |           |
                    |           ||z   }||||
|||fS )Nz Duplicate column names found: {}F-Sparse pandas data (column {}) not supported.)_resolve_columns_of_interestr   	is_unique
ValueErrorrC   list$_get_columns_to_convert_given_schema_get_index_level_valuesr   r   r   	is_sparserp   r   	enumerater   r@   pd
RangeIndex_get_range_index_descriptor)r   schemar   r   r   r   r   convert_fieldsrl   r   r   index_column_namesr   index_levelrz   	all_namess                   r6   _get_columns_to_convertr   ^  s$   *2vw??G: 
.55d2:6F6FGG
 
 	
 3BOOOL .<5-H-H))) 
 N 
" 
"h&t,, %% 	N?FFtLLN N N 	!!#&&&d###D!!!!#L11 
( 
(; a>>{KN$=>> 	,&/<<EE%%k222!!$'''E%%d+++  ''''11I |%79J,n> >r5   c                    g }g }g }g }g }g }|j         D ]}		 | |	         }
d}n# t          $ r 	 t          | |	          }
n7# t          t          f$ r# t          d                    |	                    w xY w|du r"t          d                    |	                    |At          |
t          j        j	                  r"t          d                    |	                    d}Y nw xY wt          |	          }	t          j        |
          r"t          d                    |	                    |                    |	          }|                    |
           |                    |           |                    |	           |r?|                    |	           |                    |	           |                    |
           ||z   }|||||||fS )z
    Specialized version of _get_columns_to_convert in case a Schema is
    specified.
    In that case, the Schema is used as the single point of truth for the
    table structure (types, which columns are included, order of columns, ...).
    FzNname '{}' present in the specified schema is not found in the columns or indexzlname '{}' present in the specified schema corresponds to the index, but 'preserve_index=False' was specifiedNa  name '{}' is present in the schema, but it is a RangeIndex which will not be converted as a column in the Table, but saved as metadata-only not in columns. Specify 'preserve_index=True' to force it being added as a column, or remove it from the specified schemaTr   )r   r?   _get_index_level
IndexErrorrC   r   r@   r   r   r   r   r   rp   fieldr   )r   r   r   r   r   r   r   r   r   rl   r   is_indexr   r   s                 r6   r   r     s@    LNL *% *%	T(CHH 	 	 	<&r400j) < < <..4fTll< < <<
 &&   &t. . . !(sKN$=>> ) '
 (.vd||5 5 5 HHH-	0 't,, %% 	N?FFtLLN N N T""!!#&&&e$$$D!!! 	%%%d+++$$T***$$$11I|%79J,n> >s$   
#
C$?C$4A33A.C$#C$c                     |}|| j         j        vr3t          |          r$t          |t	          d          d                   }| j                             |          S )z_
    Get the index level of a DataFrame given 'name' (column name in an arrow
    Schema).
    __index_level_)r   r   _is_generated_index_nameintr^   get_level_values)r   rl   keys      r6   r   r     s^    
 C28>!!&>t&D&D! $s+,,R/0118$$S)))r5   c                 n    	 t          j        |            | S # t          $ r t          |           cY S w xY wN)r   r   rp   rR   r   s    r6   _level_namer     sI    
4   4yys    44c                     dt          | j                  t          j        | d          t          j        | d          t          j        | d          dS )Nrangestartstopstep)kindrl   r   r   r   )r   rl   r   get_rangeindex_attribute)r   s    r6   r   r     sS     EJ''5eWEE4UFCC4UFCC  r5   c                 x     t          t           d g                    } fdt          |          D             S )Nr   c                 :    g | ]}                     |          S r4   )r   )ry   r   r   s     r6   r{   z+_get_index_level_values.<locals>.<listcomp>  s'    888!E""1%%888r5   )r^   r]   r   )r   ns   ` r6   r   r     s>    GE8eW--..A8888uQxx8888r5   c                 r     ||t          d          ||j        }n| fd|D             }n j        }|S )NzJSchema and columns arguments are mutually exclusive, pass only one of themc                 &    g | ]}|j         v |S r4   )r   )ry   cr   s     r6   r{   z0_resolve_columns_of_interest.<locals>.<listcomp>  s     999bj1r5   )r   r   r   )r   r   r   s   `  r6   r   r     sa    g1 < = = 	=		,		9999g999*Nr5   c           	         t          | d ||          \  }}}}}}}g }	|D ]}
|
j        }t          j        |          rt	          j        |
d          j        }nt          j        |          rZt          |
t          j	        j
                  r|
                    d          n	|
d d         }t	          j        |d          j        }nVt          ||
j        d           \  }}t          j                            ||          }|t	          j        |
d          j        }|	                    |           t#          || |||||	          }||	|fS )NT)from_pandasr   )r   valuesr   is_categoricalr   arrayrP   is_extension_array_dtyper@   r   Seriesheadget_datetimetz_typerO   r   _ndarray_to_arrow_typer   r   )r   r   r   r   r   _r   r   r   r   r   r   type_r   rd   s                  r6   dataframe_to_typesr     sh    ""dNG
D
DYE  %f-- 
	;HQD1116EE1&99 	;!+;>("* "* 5AFF1III/0!u HU555:EE/FFMFEF11&%@@E}555:U!Bm>5 H
 eX%%r5      Tc           	        ! t          | |||          \  }}}}	}
}}|Ht          |           t          | j                  }}||dz  k    r|dk    rt          j                    }nd}fd!d }|dk    r!fdt          ||          D             }ng }t          j        |          5 }t          ||          D ]_\  }} ||j                  r |	                     !||                     5|	                    |
                    !||                     `	 d d d            n# 1 swxY w Y   t          |          D ]6\  }}t          |t          j                  r|                                ||<   7d |D             }|Zg }t          ||          D ]3\  }}||nd}|	                    t          j        ||                     4t          j        |          }t#          || ||
|	||          }|j        rt'          |j                  nt)                      }|                    |           |                    |          }d }t          |          dk    rn	 |	d         d	         }|d
k    rH|	d         d         }|	d         d         }|	d         d         } t          t/          |||                     }n# t0          $ r Y nw xY w|||fS )Nd   r   c                    |d}d }n|j         }|j        }	 t          j        | |d          }n_# t          j        t          j        t          j        f$ r6}|xj        d                    | j	        | j
                  fz  c_        |d }~ww xY w|s@|j        dk    r5t          d                    t          |          |j                            |S )NT)rP   r   safez0Conversion failed for column {!s} with type {!s}r   z>Field {} was non-nullable but pandas column had {} null values)nullablerP   r   r   ArrowInvalidArrowNotImplementedErrorArrowTypeErrorargsrC   rl   rO   
null_countr   rR   )r   r   field_nullabler   rU   er   s         r6   convert_columnz+dataframe_to_arrays.<locals>.convert_columnI  s    =!NEE"^NJE	Xc4dKKKFF+!# 	 	 	 FFIsx336 6FFG	  	M&"3a"7"7 228&U9?9J3L 3LM M M s   1 &B1BBc                     t          | t          j                  o/| j        j        o#t          | j        j        t          j                  S r   )	r@   npndarrayflags
contiguous
issubclassrO   rP   integer)arrs    r6   _can_definitely_zero_copyz6dataframe_to_arrays.<locals>._can_definitely_zero_copy_  s;    3
++ 7	$739>2:66	8r5   c                 .    g | ]\  }} ||          S r4   r4   )ry   r   fr   s      r6   r{   z'dataframe_to_arrays.<locals>.<listcomp>e  s?     G G Ga !.A&& G G Gr5   c                     g | ]	}|j         
S r4   rP   ry   xs     r6   r{   z'dataframe_to_arrays.<locals>.<listcomp>t  s    $$$QV$$$r5   rj   r   r   r   r   r   r   )r   r^   r   r   	cpu_countr   r   ThreadPoolExecutorr   r   submitr   r@   FuturerU   r   r   r   rd   r   rx   r   with_metadatar   r   )"r   r   r   nthreadsr   r   r   r   r   r   r   r   r   nrowsncolsr  arraysexecutorr   r	  r   	maybe_futr   fieldsrl   r   pandas_metadatard   n_rowsr   r   r   r   r   s"        `                           @r6   dataframe_to_arraysr  4  s    /r6>/68 8Y 2wwBJu53;5199|~~HHH    ,8 8 8
 1}}G G G G!"4nEEG G G '11 	IX.?? I I1,,QX66 IMM..A"6"67777MM(//.!Q"G"GHHHH	I	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I &f-- 	/ 	/LAy)W^44 /%,,..q	$$V$$$E~y%00 	1 	1KD%+44DMM"(4//00006""(Bm>5 O -3OGx(((HOOO$$$!!(++F F
6{{a	$Q'/Dw)!,W5(+F3(+F3U5$5566 	 	 	D	 66!!s&   (A1D&&D*-D*AJ: :
KKc                     | j         j        t          j        k    r| |fS t	          j        |          r&|$|j        }|j        }t          j	        ||          }n|t          j
        | j                   }| |fS r   )rO   rP   r   
datetime64r   is_datetimetzrG   unitr   	timestampfrom_numpy_dtype)r   rO   r   rG   r   s        r6   r   r     sw    |BM))u} '' 2EMXzT2&&	#FL115=r5   c                 $   ddl mc m} |                     dd          }| d         }d| v rGt          j                            || d         | d                   }|                    ||          }nd	| v rQt          j	        |j
                  \  }}	t          || d	                   }
|                    |||j        |

          }nd| v r*|                    t          j        |          |          }nd| v r| d         }t          |          dk    sJ ||d                  }||         }t!          |d          st#          d          |                    |          }|                    ||          }n|                    ||          }|S )a  
    Construct a pandas Block from the `item` dictionary coming from pyarrow's
    serialization or returned by arrow::python::ConvertTableToPandas.

    This function takes care of converting dictionary types to pandas
    categorical, Timestamp-with-timezones to the proper pandas Block, and
    conversion to pandas ExtensionBlock

    Parameters
    ----------
    item : dict
        For basic types, this is a dictionary in the form of
        {'block': np.ndarray of values, 'placement': pandas block placement}.
        Additional keys are present for other types (dictionary, timezone,
        object).
    columns :
        Column names of the table being constructed, used for extension types
    extension_columns : dict
        Dictionary of {column_name: pandas_dtype} that includes all columns
        and corresponding dtypes that will be converted to a pandas
        ExtensionBlock.

    Returns
    -------
    pandas Block

    r   Nblock	placement
dictionaryr[   )r_   r[   )r%  r\   )r%  klassrO   r=   py_arrayr   __from_arrow__zGThis column does not support to be converted to a pandas ExtensionArray)pandas.core.internalscore	internalsgetr   categorical_type
from_codes
make_blockr   datetime_datarO   make_datetimetzDatetimeTZBlockpickleloadsr^   rQ   r   r)  )itemr   extension_columns_int	block_arrr%  rY   r$  r   r   rO   r  rl   pandas_dtype
pd_ext_arrs                  r6   _reconstruct_blockr<    s   8 )(((((((($''I[!It*55$|"4O 6 % % y99	t		"9?33ad:&677	Y&*&:&+   - - 
T		Y 7 7*3   5 5	t		:9~~""""y|$(.|%566 	; : ; ; ;!0055

i@@	Y??Lr5   c                     t          j                    rd} t          j                            |          }t          j        | |          S )NnsrG   )r   is_v1r   r   string_to_tzinfodatetimetz_type)r   rG   s     r6   r2  r2    sC     		 	 	$	$B&t3333r5   Fc                 b   ddl m} ddlm} g }g }|j        j        }	|s_|	]|	d         }|	                    dg           }|	d         }
t          ||	          }t          ||
||          \  }}t          |||          }n5t          j                            |j                  }t          |g |          }t          |           t          |||          }t!          | |||          }||g} |||          }t          j                    r|                    ||j                  }n ||          }|S )Nr   )BlockManager)	DataFramer   r   r   )r*  rD  pandasrE  r   r  r-  _add_any_metadata_reconstruct_index_get_extension_dtypesr   r   r   num_rows'_check_data_column_metadata_consistency_deserialize_column_index_table_to_blocks	is_ge_v21	_from_mgraxes)optionstabler_   ignore_metadatatypes_mapperrD  rE  all_columnsr   r  r   r   ext_columns_dtypesr   blocksrP  mgrr   s                     r6   table_to_dataframerY    sr    322222      KNl2O L:%i0(,,-=rBB+O<!%99)%1B*5|E Eu2;. . ))%.9925"lKK+K888'{NKKGguj:LMMFUD
,vt
$
$C   ch//Ys^^Ir5   c                 P    h | ]#}t          t          j        |                    $S r4   )rR   r   rO   )ry   typs     r6   	<setcomp>r\    s8     ! ! ! ! ! !r5   r   uintfloatr=   c                 &   i }t           j        |S |D ]x}	 |d         }n# t          $ r |d         }Y nw xY w|d         }|t          vrCt          j        |          }t          |t           j                  rt          |d          r|||<   y| j        D ]S}|j        }	t          |	t          j
                  r0	 |	                                }|||j        <   C# t          $ r Y Ow xY wT|r(| j        D ] }|j        }	 ||	          }|
|||j        <   !|S )a  
    Based on the stored column pandas metadata and the extension types
    in the arrow schema, infer which columns should be converted to a
    pandas extension dtype.

    The 'numpy_type' field in the column metadata stores the string
    representation of the original pandas dtype (and, despite its name,
    not the 'pandas_type' field).
    Based on this string representation, a pandas/numpy dtype is constructed
    and then we can check if this dtype supports conversion from arrow.

    Nrm   rl   ro   r)  )r   extension_dtyper?   _pandas_supported_numpy_typesr:  r@   rQ   r   rP   r   BaseExtensionTypeto_pandas_dtyperl   NotImplementedError)
rR  columns_metadatarT  ext_columnscol_metarl   rO   r:  r   r[  s
             r6   rI  rI    s    K "* % 5 5	$L)DD 	$ 	$ 	$F#DDD	$&555 '3E::L,(CDD 5<)9:: 5(4K%  7 7jc2/00 	77"2244 +7EJ'' '   	7  7\ 	7 	7E*C'<,,L'*6EJ's   447C
C#"C#c                 <    t          d | D                       sJ d S )Nc              3   F   K   | ]}|d          du rd|v p	|d          duV  dS )rl   Nrm   r4   ry   r   s     r6   	<genexpr>z:_check_data_column_metadata_consistency.<locals>.<genexpr>Z  sW         
6d		0|q0JQvYd5J     r5   )all)rU  s    r6   rK  rK  U  sF    
            r5   c                    d | j         D             }|rd |D             fd|D             }n|}t          |          dk    rt          j        nd }|s t          j                            |          }nMt          j        j                            t          t          ||                    d |D             pd           }t          |          dk    rt          ||          }t          |          }|S )	Nc                 Z    g | ](}t          |t                    rt          |          n|)S r4   )r@   r   r	   r  s     r6   r{   z-_deserialize_column_index.<locals>.<listcomp>a  sA     9 9 9 '1E&:&:Ailll 9 9 9r5   c           	      n    i | ]2}|                     d t          |d                             |d         3S rm   rl   )r-  r   rj  s     r6   
<dictcomp>z-_deserialize_column_index.<locals>.<dictcomp>d  sI     
 
 
 EE, 7&	 B BCCQvY
 
 
r5   c                 <    g | ]}                     ||          S r4   r-  )ry   rl   columns_name_dicts     r6   r{   z-_deserialize_column_index.<locals>.<listcomp>h  s7     
 
 
26!!$--
 
 
r5   r   c                     | fS r   r4   )r  s    r6   <lambda>z+_deserialize_column_index.<locals>.<lambda>q  s    ! r5   c                     g | ]
}|d          S r   r4   )ry   	col_indexs     r6   r{   z-_deserialize_column_index.<locals>.<listcomp>{  s    EEE9V$EEEr5   r   r   )r   r^   astliteral_evalr   r   Index
MultiIndexfrom_tuplesr   r   "_reconstruct_columns_from_metadata _flatten_single_level_multiindex)block_tablerU  r   column_stringscolumns_valuesto_pairr   rt  s          @r6   rL  rL  `  sB   9 9*79 9 9N 	(
 
 
 
 

 
 
 
:H
 
 
 (
 #&n"5"5"9"9c~~G
  
.&&~66.+77Wn--..EEnEEEM 8 
 
 >Q4WnMM /w77GNr5   c                 Z   d |D             }g }g }| }|D ]}t          |t                    rt          | ||||          \  }}	}
|	1n|d         dk    r_|d         }
t          j                            |d         |d         |d         |
          }	t          |	          t          |           k    rn(t          d	                    |d                             |	                    |	           |	                    |
           t          j        }t          |          d
k    r|j
                            ||          }nht          |          d
k    r;|d         }t          ||j                  s|                    ||d                   }n|                    | j                  }||fS )Nc                 H    i | ]}|                     d |d                   | S rp  rs  rj  s     r6   rq  z&_reconstruct_index.<locals>.<dictcomp>  s<        	
lAfI&&  r5   r   r   rl   r   r   r   )r   rl   zUnrecognized index kind: {}r   ry  r   r   )r@   rR   _extract_index_levelr   r   r   r^   r   rC   r   r}  from_arraysr|  rJ  )rR  r   rU  rT  field_name_to_metadataindex_arraysindex_namesresult_tablerz   r   
index_namer   r   s                r6   rH  rH    s       LKL" ' 'eS!! 	54H|U,BL5R 5R1L+z" # 6]g%%vJ%.33E'N49&M9>v9C 4 E EK ;3u::-- . :$fU6]335 5 5K(((:&&&&	B <1)),k)JJ	\		a		Q%** 	9HHUQH88Een--r5   c                    ||         d         }t          ||          }| j                            |          }|dk    r|d d fS t          j        }|                     |          }	|	                    |          j        }
t          |
d          r |
j	        j
        s|
                                }
t          |	j        t          j        j                  r<|	j        j        0t%          |                    |
d          |	j        j                  }n|                    |
|
j        d          }|                    |j                            |                    }|||fS )Nrl   )rT  r  F)copy)rO   r  ) _backwards_compatible_index_namer   get_field_indexr   r   rb   	to_pandasr   rQ   r  	writeabler  r@   rP   r   r   rF   rG   make_tz_awarer   rO   remove_column)rR  r  rm   r  rT  logical_namer  r   r   r   r   r   s               r6   r  r    s=   )*5f=L1*lKKJ$$Z00ABwwT4''	B
,,q//C]]]55<Fvw (>  #(BF011 Hchk6M#BIIf5I$A$A38;OOiifliGG--++J77 L j00r5   c                 4    | |k    rt          |           rdS |S )a1  Compute the name of an index column that is compatible with older
    versions of :mod:`pyarrow`.

    Parameters
    ----------
    raw_name : str
    logical_name : str

    Returns
    -------
    result : str

    Notes
    -----
    * Part of :func:`~pyarrow.pandas_compat.table_to_blockmanager`
    N)r   )raw_namer  s     r6   r  r    s(    $ <$<X$F$Ftr5   c                 4    d}t          j        ||           d uS )Nz^__index_level_\d+__$)rematch)rl   patterns     r6   r   r     s    &G8GT""$..r5   rM   )	r   r;   r:   r   r   rK   r  floatingr   c                     	 t           |          S # t          $ r) d| v rt          j        cY S t          j        |           cY S w xY w)a  Get the numpy dtype that corresponds to a pandas type.

    Parameters
    ----------
    pandas_type : str
        The result of a call to pandas.lib.infer_dtype.

    Returns
    -------
    dtype : np.dtype
        The dtype that corresponds to `pandas_type`.
    r   )_pandas_logical_type_mapr?   r   object_rO   )rn   s    r6   _pandas_type_to_numpy_typer    s\    %'44 % % %k!!:x$$$$$	%s    AAAc                 R    t          | t          j        j                  r| j        S d S r   )r@   r   r   r}  r`   )mis    r6   _get_multiindex_codesr    s%    "kn/00 xtr5   c                    t           j        t          | dd          p| g}t          |           pfd|D             }d t	          ||i           D             }g }t          j        dd          }|D ]\  }}}	t          |          }
|
t          j	        k    r|
                    |          }|dk    r\t          j                            |d	         d
         d                   }                    |d                              |          }n |j        |
k    r|                    |
          }|j        |	k    r|dk    r|                    |	          }|                    |                               ||| j                  S )a_  Construct a pandas MultiIndex from `columns` and column index metadata
    in `column_indexes`.

    Parameters
    ----------
    columns : List[pd.Index]
        The columns coming from a pyarrow.Table
    column_indexes : List[Dict[str, str]]
        The column index metadata deserialized from the JSON schema metadata
        in a :class:`~pyarrow.Table`.

    Returns
    -------
    result : MultiIndex
        The index reconstructed using `column_indexes` metadata with levels of
        the correct type.

    Notes
    -----
    * Part of :func:`~pyarrow.pandas_compat.table_to_blockmanager`
    r   Nc                 T    g | ]$}                     t          |                    %S r4   )r   r^   )ry   r   r   s     r6   r{   z6_reconstruct_columns_from_metadata.<locals>.<listcomp>=  s8     0 0 0&+c%jj!!0 0 0r5   c           	          g | ]D\  }}||                     d t          |j                            |                     dd          fES )rn   ro   N)r-  rR   rO   )ry   r   rx  s      r6   r{   z6_reconstruct_columns_from_metadata.<locals>.<listcomp>B  s^        E9 
	mS-=-=>>	|T	*	*	,  r5   )	fillvaluer   r   r:   r   rd   r\   T)utcry  )r   r   r]   r  r   operatormethodcallerr  r   bytes_r   r   r   rA  to_datetime
tz_convertrO   astyper   r}  r   )r   r   r   labelslevels_dtypes
new_levelsencoderr   r:  numpy_dtyperO   rG   r   s               @r6   r  r  #  s   , 
B Wh--:'F"7++  0 0 0 0/50 0 0F
  !,Nb!
 !
 !
  M J#Hg66G,9 ! !(|[*<88 BIIIg&&E<''((q!*-j9; ;BNN5dN33>>rBBEE[E!!LL''E;+%%,,*F*FLL--E%    ==V7==AAAr5   c           	          |j         t          j                            | ||t	                                                              }fd|D             S )Nc                 2    g | ]}t          |          S r4   )r<  )ry   r6  r   r7  s     r6   r{   z$_table_to_blocks.<locals>.<listcomp>k  s6           tW.?@@      r5   )r   r   r   table_to_blocksr   keys)rQ  r  r_   r7  rU   r   s      ` @r6   rM  rM  d  su     &GV##G[*$():)?)?)A)A$B$BD DF                r5   c                 ,   t           j        }t          | |j                  rq| j        dk    rf| j        \  t          |           \  }j        }| j        st          d          |
                    fd|D             || j        d                   S | S )Nr   zFound non-unique column indexc                 0    g | ]}|d k    r|         ndS )r  Nr4   )ry   _labelr   s     r6   r{   z4_flatten_single_level_multiindex.<locals>.<listcomp>|  s)    KKK&v||VF^^KKKr5   r   )rO   rl   )r   r   r@   r}  nlevelsr   r  rO   r   r   r|  r   )r   r   r  rO   r   s       @r6   r  r  o  s    	B%'' 
EMQ,>,>,'..  	><===xxKKKKFKKKQ  
 
 	

 Lr5   c                    i }i }| j         }|d         }d |D             }t          |          }t          |d                   |z
  }t          |d                   D ]3\  }}	|	                    d          }
|
s|	d         }
||k    r|||z
           }
|
d}
|                    |
          }|dk    r|	d         d	k    r| |         }t          |j        t          j        j	                  s|	d
         }|s|                    d          }|r||j        j
        k    rs|                                }t          j        d|          }t          j                            ||          }t          j        ||         j        |          ||<   |||<   5t          |          dk    rg }g }t#          t          | j                             D ]x}||v r7|                    ||                    |                    ||                    =|                    | |                    |                    | j         |                    yt          j                            |t          j         |                    S | S )Nr   c                 <    g | ]}t          |t                    |S r4   )r@   rR   )ry   idx_cols     r6   r{   z%_add_any_metadata.<locals>.<listcomp>  s6     2 2 2"7C002W 2 2 2r5   r   rm   rl   rj   r  rn   r:   rd   r\   r>  r?  r  r   )r   )r   r^   r   r-  r  r@   rP   r   r   rF   rG   r  r!  Arrayr   r   rl   r   r   Tabler  )rR  r  modified_columnsmodified_fieldsr   r   n_index_levels	n_columnsr   rg  r  idxr   rd   metadata_tz	convertedtz_aware_typer  r   r  s                       r6   rG  rG    s   O\F#O4M2 2M 2 2 2M''NOI.//.@I !!;<< : :8<<-- 	"'HI~~(Y7!$$X.."99&,66Cj!#(BF,@AA #J/ &ll:66 :;#(+#=#= #I$&L+$F$F$FM$&H$8$8>K %9 %M %MM ,.8F3K4D4A,C ,COC(,9$S)
q  s5<(()) 	/ 	/A$$$/2333oa01111uQx(((el1o....x##GBIf4E4E#FFFr5   c                     t           j                            |          }| j                            d          j                            |          } | S )zB
    Make a datetime64 Series timezone-aware for the given tz
    r  )r   r   rA  dttz_localizer  )seriesrG   s     r6   r  r    sC     
	 	 	$	$Bi##E**

2 Mr5   r   )r   NT)NN)NFN)Orz  collections.abcr   
concurrentr   concurrent.futures.threadr  r   	itertoolsr   r   r  r4  r  r   numpyr   numpy.core.numerictypesr   _np_sctypesr   r   pyarrow.libr   r	   r   r7   rD   bool_r   r   r   r   r   r   r   r   r   r   str_r  rN   rV   rf   ru   r   r   r   r   r   r   r   r   r   r   r   r   r  r   r<  r2  rY  ra  rI  rK  rL  rH  r  r  r   r  r  r  r  r  rM  r  rG  r  r4   r5   r6   <module>r     s:  & 


 $ $ $ $ $ $       !             ! ! ! ! ! !    				      : : : : : :     . . . . . . . .    :  $ HfGVHgHgHgHgIxIxIxJ	J	VGXIw $  $ $ $&* * *ZS S Sl  ($ $ $N0 0 0&=> => =>@=> => =>@
* 
* 
*    9 9 9
  & & & &D IM!_" _" _" _"D  &< < < <~4 4 4 JN! ! ! !L! !E"[%88;w;OO6"#! ! ! 6 6 6 6r  % % %P3 3 3 3n ?C1 1 1 1<  0/ / /  "wYgx
Z
 
 % % %,  >B >B >BB       (: : :B    r5   