
    >ie                   r   d dl mZ d dlZd dlZd dlZd dlZd dlmZ d dlmZ d dl	m
Z
 d dlZd dlZd dlZd dlmZ d dlmZmZ d dlmZ d dlmZ d d	lmZ d dlZd d
lmZ d dl m!Z! d dl"m#Z# d dl$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z- d dl.m/Z/m0Z0m1Z1 d dl2m3Z3m4Z4 d dl5m6Z6 d dl7m8Z8m9Z9  ej:                     ej;                     ej<                     ej=                     ej>                     ej?                     ej@                     ejA                     ejB                     ejC                     ejD                     ejE                     ejF                     ejG                     ejH                     ejI                     ejJ                     ejK                     ejL                     ejM                     ejN                     ejO                     ejP                     ejQ                    iZRd ZSd ZT	 	 d%dZUd ZV G d d          ZWd ZXd ZYd ZZd Z[d Z\d Z]d&d Z^d'd"Z_ G d# d$e%          Z`dS )(    )annotationsN)defaultdict)datetime)reduce)expand_paths_if_neededstringify_path)ArrowFSWrapper)datasetfs)tokenize)flatten)pyarrow_schema_dispatch)	Engine_get_aggregation_depth_infer_split_row_groups_normalize_index_columns_process_open_file_options_row_groups_to_parts_set_gather_statistics_set_metadata_task_size_sort_and_analyze_paths)_get_pyarrow_dtypes_is_local_fs_open_input_files)clear_known_categoriespyarrow_strings_enabled)Delayed)
getargspecnatural_sort_keyc                >    t          | t                    r| j        n| S )z5Return the wrapped filesystem if fs is ArrowFSWrapper)
isinstancer	   r   r   s    ?lib/python3.11/site-packages/dask/dataframe/io/parquet/arrow.py_wrapped_fsr$   ?   s    r>22:255:    c                    	 |                      |           dS # t          $ r(}dt          |          v rt          d          ||d}~ww xY w)zmAppend row-group metadata and include a helpful
    error message if an inconsistent schema is detected.
    zrequires equal schemaszSchemas are inconsistent, try using `to_parquet(..., schema="infer")`, or pass an explicit pyarrow schema. Such as `to_parquet(..., schema={"column1": pa.string()})`N)append_row_groupsRuntimeErrorstr)metadatamderrs      r#   _append_row_groupsr-   D   su    ""2&&&&& 	 	 	#s3xx//E 
  I	s    
A#AA Tc
                    |                     |d           |r                    d           | j        j                 |rt	          |          ng }d}|r                    |d           d}fd|D             }                    |d          }j                            |          }t          |          dk    r|st          d	          | j        }| j        j        D ].}||v r(|
                    |                    |                    }/g }t          |          d
k    r|d         n|}|                    |dd          }|D ]+\  }}t          |t                    s|f}|j                            d t#          ||          D                       } ||||          }|j                            ||g          }|                     |d           |j                            ||g          }|                    |d          5 }t'          j        ||fd|	r|ndi|
 ddd           n# 1 swxY w Y   |	r5|d                             |j                            ||g                     -|S )zWrite table to a partitioned dataset with pyarrow.

    Logic copied from pyarrow.parquet.
    (arrow/python/pyarrow/parquet.py::write_to_dataset)

    TODO: Remove this in favor of pyarrow's `write_to_dataset`
          once ARROW-8244 is addressed.
    Texist_okinplaceFc                     g | ]
}|         S r.   r.   ).0coldfs     r#   
<listcomp>z&_write_partitioned.<locals>.<listcomp>w   s    888#bg888r%   columns)axisr   z.No data left to save outside partition columns   )dropnaobservedc                4    g | ]\  }}t          ||          S r.   )_hive_dirname)r5   namevals      r#   r8   z&_write_partitioned.<locals>.<listcomp>   s&    QQQ)$]4%%QQQr%   preserve_indexschemawbmetadata_collectorN)mkdirsreset_indexrD   nameslist	set_indexdropr9   len
ValueErrorremoveget_field_indexgroupbyr"   tuplesepjoinzipopenpqwrite_tableset_file_path)tabler7   	root_pathfilenamepartition_colsr   pandas_to_arrow_tablerC   
index_colsreturn_metadatakwargspartition_keysdata_df	data_cols	subschemar6   md_listgbkeyssubgroupsubdirsubtableprefix	full_pathfs    `                       r#   _write_partitionedrp   V   s   * IIi$I''' %
t$$$	EL	B%/7j!!!RJN 
Z...8888888Nggn9g55G
//I
9~~:IJJJI|! I I.  !(()B)B3)G)GHHIG*-n*=*=*B*B^A&&N		F	FB G Gh$&& 	7DQQs>47P7PQQQ
 
 )(^I
 
 
 i011
		&4	(((FKK 233	WWY%% 	N  />#G774 	  	 	 	 	 	 	 	 	 	 	 	 	 	 	 	  	GBK%%bfkk682D&E&EFFFNs   H::H>	H>	c                    | rG|Et          t          |                               |j                            t          |           k    S | rdS dS )z[Simple utility to check if all `index` columns are included
    in the known `schema`.
    NTF)rN   setintersectionrJ   )indexrD   s     r#   _index_in_schemaru      sR      #3u::**6<8899SZZGG	 tur%   c                      e Zd ZdZd ZdS )PartitionObjag  Simple class providing a `name` and `keys` attribute
    for a single partition column.

    This class was originally designed as a mechanism to build a
    duck-typed version of pyarrow's deprecated `ParquetPartitions`
    class. Now that `ArrowLegacyEngine` is deprecated, this class
    can be modified/removed, but it is still used as a convenience.
    c                n    || _         t          j        |                                d          | _        d S )NF)copy)r@   pdIndexsort_valuesri   )selfr@   ri   s      r#   __init__zPartitionObj.__init__   s/    	HT--//e<<<			r%   N)__name__
__module____qualname____doc__r~   r.   r%   r#   rw   rw      s-         = = = = =r%   rw   c                \    | j                             | j        | j        | j        |          S )z*Create new fragment with row-group subset.)
row_groups)formatmake_fragmentpath
filesystempartition_expression)old_fragr   s     r#   _frag_subsetr      s5    ?((%	 )   r%   c                    | j         duod| j         v }|r2t          j        | j         d                             d                    S i S )z)Get pandas-specific metadata from schema.N   pandasutf8)r*   jsonloadsdecode)rD   has_pandas_metadatas     r#   _get_pandas_metadatar      sO     !/5V)v:V z&/)4;;FCCDDD	r%   c           	     N   |                     di                                           }t          |                    di           fi t	          |          rdddn||dgk    r|n|gddd\  }}	|                     d	d          du }
d
|                    d
|
          i}t          | gf||d|	d         5 }|dgk    r- t          j        |fi |j        d|ddd|cddd           S  t          j        |fi |j	        |f|ddd|cddd           S # 1 swxY w Y   dS )zRead arrow table from file path.

    Used by `ArrowDatasetEngine._read_table` when no filters
    are specified (otherwise fragments are converted directly
    into tables).
    readopen_file_optionsFnone)allow_precachedefault_cacheNpyarrow)r9   r   default_enginer   method
pre_buffer)r   precache_optionsr   T)r9   use_threadsuse_pandas_metadatar.   )
getry   r   popr   r   rX   ParquetFiler   read_row_groups)r   r   r   r9   rD   filtersrb   read_kwargsr   r   pre_buffer_defaultr   fils                r#   _read_table_from_pathr      s6   " **VR((--//K*D+R00+ + B	
"'!'   #,64&,@,@jjzl"+!'	 + +''( *--h==E>P Q QRJ		
)
 
 	
 

 
	 
 $92>#44449 !$(  	         E2>#4444D!$(	 
                   s   7(D,!DD!Dc                     d t           j        j                  D              fd}d t          ||          D             S )aq  Custom version of pyarrow's RowGroupInfo.statistics method
    (https://github.com/apache/arrow/blob/master/python/pyarrow/_dataset.pyx)

    We use column names to specify the specific subset of columns
    that we need statistics for.  This is more optimal than the
    upstream `RowGroupInfo.statistics` method, which will return
    statistics for all columns.
    c                    i | ]\  }}||	S r.   r.   )r5   icol_names      r#   
<dictcomp>z&_get_rg_statistics.<locals>.<dictcomp>  s+       #8!  r%   c                    j                             |                    }|j        }||j        sdS |j        }j                            |          }|dk     rdS |j        |j        |j        |j	        dfS )N)NNr   )minmax
null_count)
r*   column
statisticshas_min_maxpath_in_schemarD   rQ   r   r   r   )column_namer6   statsr@   field_index	row_grouprow_group_schemas        r#   
name_statsz&_get_rg_statistics.<locals>.name_stats  s     ''(8(EFF= 1=:!&66t<<??:!99*$
 $
 
 	
r%   c                    i | ]
\  }}|||S Nr.   )r5   r@   r   s      r#   r   z&_get_rg_statistics.<locals>.<dictcomp>.  s*       #eUEVeEVEVEVr%   )	enumeraterD   rJ   map)r   	col_namesr   r   s   `  @r#   _get_rg_statisticsr     sz     '01A1G'H'H  
 
 
 
 
 
$ '*:y'A'A   r%   c                    |r d t          |t                    D             nt                      }| r d t          | t                    D             nt                      }t          ||z
            S )Nc                <    h | ]}t          |          |d          S r   rN   r5   vs     r#   	<setcomp>z"_need_filtering.<locals>.<setcomp>9  s'    III!#a&&I1IIIr%   	containerc                <    h | ]}t          |          |d          S r   r   r   s     r#   r   z"_need_filtering.<locals>.<setcomp>>  s'    BBB!3q66B1BBBr%   )r   rK   rr   bool)r   rc   r^   filtered_colss       r#   _need_filteringr   3  s     	IIw~>>>IIIIUU  GNXBBww$777BBBBSVSXSX  .///r%   c                @    t          j        |          rdn|}|  d| S )N__HIVE_DEFAULT_PARTITION__=)rz   isna)r@   rA   s     r#   r?   r?   D  s,     +-'#,,
?
&
&CC??S??r%   c                Z    dt          | t                    rt          j        di | n| i|S )Npartitioningr.   )r"   dictpa_dsr   )r   rb   s     r#   _process_kwargsr   L  sF    
 	,--E.....	  r%   Fc                   t          | t          j                  r| S | _t          |           dk    st	          d | D                       rt          d          t          | d         d         t                    r| g} fdg }| D ]=}fd|D             }|                    t          t          j
        |                     >t          t          j        |          S )Nr   c              3  <   K   | ]}t          |          d k    V  dS )r   Nr   )r5   ro   s     r#   	<genexpr>z)_filters_to_expression.<locals>.<genexpr>d  s,      #A#AACFFaK#A#A#A#A#A#Ar%   zMalformed filtersc                v   t          j        |           }|r]|t          j        u rO|dk    r|                              S |dk    r|                               S t          d| ||f d          |dk    s|dk    r||k    }n|dk    r||k    }n|d	k    r||k     }nv|d
k    r||k    }ni|dk    r||k    }n\|dk    r||k    }nO|dk    r|                    |          }n3|dk    r|                    |           }nt          d| ||f d          s|dv r|                              |z  S |S )Nis)nan_is_nullzis not"zO" is not a supported predicate Please use "is" or "is not" for null comparison.r   z==!=<>z<=z>=innot inz(" is not a valid operator in predicates.)r   r   )r   fieldnpnanis_nullrO   isin)r6   oprA   r   exprr   propagate_nulls        r#   convert_single_predicatez8_filters_to_expression.<locals>.convert_single_predicatel  s   C   ;;;3"&==Tzz}}}===x+>>>> Hb# H H H  
 99d

C<DD4ZZC<DD3YY3;DD3YY3;DD4ZZC<DD4ZZC<DD4ZZ::c??DD8^^JJsOO#DDLS"cNLLL  
  	A"(8"8"8==[=99D@@r%   c                2    g | ]\  }}} |||          S r.   r.   )r5   r6   r   rA   r   s       r#   r8   z*_filters_to_expression.<locals>.<listcomp>  s<     
 
 
7CsB$$S"c22
 
 
r%   )r"   r   
ExpressionrN   anyrO   r)   appendr   operatorand_or_)r   r   r   disjunction_membersconjunctionconjunction_membersr   s    ``   @r#   _filters_to_expressionr   Z  s     '5+,, w<<1#A#A#A#A#A A A0111gajmS)) 	  iG' ' ' ' ' 'R  O O
 
 
 
GR
 
 
 	""6(-9L#M#MNNNN(, 3444r%   c                     e Zd Zed             Ze	 	 	 	 	 	 	 	 	 	 	 	 dd            Zed             Ze	 	 	 	 	 d d	            Ze	 	 	 	 	 	 d!d            Ze	 d"d#d            Z	e	 	 	 	 	 	 d$d            Z
ed%d            Zed             Zed             Zed             Zed             Ze	 	 	 	 d&d            Zed             Zedddd            Ze	 	 d'd(d            Zed             Zed             ZdS ))ArrowDatasetEnginec                "   ||                     dd          }nd|v rt          d          |}t          |t          j                  s|dv r*t          |t
          t          t          f          r|st          d          d |D             }nt          |          g}|dv rSt          j        	                    |d                   d         }|r&d|v ri nd|j
        i} t          |          di ||}t          |          }|d                             d	          r+t          |t          j                  rdd
lm}	  |	            n|t!          |dd|d           }
|fd|
D             |d|j        ifS t%          j        |||||          S )Nr   fsspeczXCannot specify a filesystem argument if the 'filesystem' dataset option is also defined.)arrowr   zempty urlpath sequencec                ,    g | ]}t          |          S r.   )r   )r5   us     r#   r8   z9ArrowDatasetEngine.extract_filesystem.<locals>.<listcomp>  s     >>>>!,,>>>r%   r   regionzC:)LocalFileSystemrbr;   c                :    g | ]}                     |          S r.   )_strip_protocol)r5   r   fs_strips     r#   r8   z9ArrowDatasetEngine.extract_filesystem.<locals>.<listcomp>  s'    <<<))!,,<<<r%   open_file_funcr.   )r   rO   r"   pa_fs
FileSystemrK   rS   rr   r   from_urir   typer	   
startswithr  fsspec.implementations.localr   open_input_filer   extract_filesystem)clsurlpathr   dataset_optionsr   storage_optionsr   r   	fsspec_fsr  pathsr  s              @r#   r  z%ArrowDatasetEngine.extract_filesystem  s     $$\8<<BB.. C   B b%*++  	r5I/I/I'D%#566 4 ?$%=>>>>>g>>>)'223)))%..wqz::1=" ? '/99")?T  "b>>F>o>>B&r**Iqz$$T** %z"e>S/T/T % IHHHHH*?,,$*7D!YMME<<<<e<<<!2#56	  (
 
 	
r%   NadaptiveFr   c                   ||t          d          |Et          j        dt                     |r(t          j                            dd           }|dv rd}|}|dvrt          d|           ||d	<   |                     |||||||	|
|||||          }|                     |          }| 	                    |          \  }}}t          |          r-||d
         d<   |d         |d
         d<   |d         |d
         d<   ||||d         fS )NzR`use_nullable_dtypes` is deprecated. Use `dtype_backend` keyword argument instead.zThe `use_nullable_dtypes=` keyword argument and `dataframe.dtype_backend` config option are deprecated, and will be removed in the future. Use the `dtype_backend=` keyword argument instead.)categoryzdataframe.dtype_backend)Npandasnumpy_nullable)Nr  r   zJ`dtype_backend` should be one of [None, 'numpy_nullable', 'pyarrow'], got dtype_backendr   common_kwargsaggregation_depthsplit_row_groupsrt   )rO   warningswarnFutureWarningdaskconfigr   _collect_dataset_info_create_dd_meta_construct_collection_planrN   )r  r   r  
categoriesrt   use_nullable_dtypesr  gather_statisticsr   r  	blocksizeaggregate_filesignore_metadata_filemetadata_task_sizeparquet_file_extensionrb   config_backenddataset_infometapartsr   r  s                         r#   read_metadataz ArrowDatasetEngine.read_metadata  s   & *}/Hd   *ME '	    # /!%1JD!Q!Q!%555%5N . CCCl]jll   #0 00 "
 
" ""<00 '*&D&D\&R&R#um
 u:: 	L(5E!H_%,89L,ME!H()+78J+KE!H'(eUL$9::r%   c                    | t           k    S r   )r   )r  s    r#   multi_supportz ArrowDatasetEngine.multi_support8  s    (((r%   r.   c
                p   t          |t                    r|D ]}||vr|                    |           |                                }t          |t          t          f          sA|r?|r=|j        D ]1}||v r|                    |           |                    |           2|pd}t          |t                    s|g}g }t          |          dk    }|D ]}t          |t                    r|}d}d}n|\  }}}t          |t                    rt          |          }t          |t                    s|g} | j	        |||||	|||fi |
}|r|                    |           |rt          j        |          } | j        ||fd|i|
}|rt          |t                    r|D ]}}t          |j                  rg||j                 j        j        dk    rLt!          j        t!          j        |j        ||j                 j                  |j                  ||j        <   ~t+          |j        j                                      t+          |                    }|s2|r|                    dd	           n|                    dd	           n~t+          |j        j                  t+          |          k    r|r|                    dd	           n:|r8d}t          t+          |          t+          |j        j                  z
            }|t          |                   }|r|                    |          }|S )
z!Read in a single output partitionNr;   r  r  r&  valuesrt   FT)rM   r3   )r"   rK   r   ry   rS   partition_namesrP   rN   r)   _read_tablepaconcat_tables_arrow_table_to_pandasri   r@   dtyperz   SeriesCategoricalr7  rt   rr   rJ   issubsetrI   rL   )r  r   piecesr9   rt   r  r&  
partitionsr   rD   rb   levelcolumns_and_parts	part_nametables
multi_readpiecepath_or_fragr   rc   arrow_tabler7   	partitionindex_in_columns_and_partss                           r#   read_partitionz!ArrowDatasetEngine.read_partition<  s     eT"" 	* * * ''NN5))) $LLNN*tUm44 	*  *Z *!+!; < <I G++y1111)00;;;;!/T &$'' 	XF[[1_
 	+ 	+E%%% B$ 	!% =B9y. )U++ , OO	i.. (&K	 *#/
 
 
 
K  +k*** 	3*622K (S'
 
3@
DJ
 
  	*Z66 	'  	y~&& 
2in+=+C+HJ+V+V *,'0~#%in#5#<   !h* * *By~& &)%8%8%A%A!""&
 &
"  	W) 8 E48888 D$777728>""c%jj005O0 E48888+ W$(->)?)?#bhnBUBU)U$V$V!&''( 	%e$$B	r%   inferc
           	     J	  ! |dk    st          |t                    rt          |	r|j                            |	          n|j                                                  }t          |t                    rqt          j        |          }|j        D ]U}|	                    |          }|	                    |          }|
                    ||                    |                    }V|}|                    |d           |r|d}d }d !d}d}|r't          j        |t          |          d          }t!          |j                  }|dk    r	 |                    |j                            |dg          d	
          5 }t+          j        |          }d d d            n# 1 swxY w Y   |!d}n# t.          $ rw 	 |                    t1          |j        t2                    d         d	
          5 }t+          j        |          !d d d            n# 1 swxY w Y   n# t.          $ r Y nw xY wY nw xY wd}|ra!^!j                                        }|j        }|j        d uod|j        v }|rEt9          j        |j        d                             d                    }d |d         D             }nd }t?          ||          }t          |          t          |j                   t          |          z
  k    r5tC          d"                    |tG          |j                                       tI          j%        |          j&        |         ||         j'        k    (                                rhtC          d"                    t          |)                                          t          |j'        )                                          z                      |d         |vrd}|sd }!fdtU          !j+                  D             }|,                    |d                   }|D ]N}|-                    |          }|j.        r0||j.        j/        }-|j.        j/        |k    r|j.        j/        }Jd } nO|d         }||d         |k    rtC          d          ||	d} |||| fS )NrO  Tr0   r   Fparquet)r   r   	_metadatar  )modekeyrG   r   r   c                6    g | ]}|d          dk    |d         S )pandas_typecategoricalr@   r.   )r5   cs     r#   r8   z7ArrowDatasetEngine.initialize_write.<locals>.<listcomp>  s6       '=88 fI888r%   r9   z5Appended columns not the same.
Previous: {} | New: {}zAppended dtypes differ.
{}r@   c              3  B   K   | ]}                     |          V  d S r   )r   )r5   r   tail_metadatas     r#   r   z6ArrowDatasetEngine.initialize_write.<locals>.<genexpr>  sE         "++A..     r%   	divisionszThe divisions of the appended dataframe overlap with previously written divisions. If this is desired, set ``ignore_divisions=True`` to append anyway.
- End of last written partition: {old_end}
- Start of first new partition: {divisions[0]})rD   r`   )0r"   r   r   _meta_nonemptyrL   remove_metadatar;  rD   rJ   rQ   rr   r   rH   r   r
   r$   rN   filesrW   rT   rU   rX   r2  OSErrorsortedr    to_arrow_schemar*   r   r   r   r   r9   rO   r   rK   rz   r?  locdtypesr   itemsrangenum_row_groupsrt   r   r   r   )"r  r7   r   r   r   partition_onignore_divisionsdivision_inforD   r`   rb   inferred_schemar@   r   jfull_metadatai_offsetmetadata_file_existsdsr   arrow_schemarJ   r   pandas_metadatar&  rd  old_endr   index_col_ir   r   r\  extra_write_kwargsr[  s"                                    @r#   initialize_writez#ArrowDatasetEngine.initialize_write  s    W
64 8 85'!++J777&  o	  &$'' N6**"L N ND'77==A..t44A&5&9&9!V\\!__&M&MOO$F 			$	&&& 	$m+#$ 	tB	RRRB28}}H!||dK-@!A!AMM >QT(*(8(=(=> > > > > > > > > > > > > > >$1M+/((   WW"281ABBB2FT %   B ,.,<S,A,AMB B B B B B B B B B B B B B B #      >	m/(/??AAL &E%T1Xi<CX6X   # 
""&* ))4;;FCC# # ,Y7  

 "
(zBBF5zzS__s</@/@@@@ --3VE4
;K;K-L-L   )F##'."U)2BBGGII  188FLLNN++c")//2C2C.D.DD    V$E11#' #    "=#?@@  
 $kk-*?@@!+ " "I&--k::F( 	""?&,&7&;GG#.2W<<&,&7&;GG '+G!E	" *+6	&9Q<7+B+B$I   )/jII(<>PPPs~   1F0 ?F F0  F$$F0 'F$(F0 0
H1;6H1HHH	HH	HH1
H+(H1*H++H10H1r7   pd.DataFramereturnpa.Tablec           
        	 t           j                            |d||          S # t           j        $ r}| t           j                            |          }t          j        |                    d          d          }t          j        |                    d          d          }t          d|d| d| d	          d d }~ww xY w)
Nr;   )nthreadsrC   rD   F)show_schema_metadataz    z=Failed to convert partition to expected pyarrow schema:
    `z`

Expected partition schema:
z

Received partition schema:
z

This error *may* be resolved by passing in schema information for
the mismatched column(s) using the `schema` keyword in `to_parquet`.)	r;  Tablefrom_pandasArrowExceptionSchematextwrapindent	to_stringrO   )r  r7   rC   rD   exc	df_schemaexpectedactuals           r#   _pandas_to_arrow_tablez)ArrowDatasetEngine._pandas_to_arrow_table;  s   	8''Q~f (      	 	 	~	--b11I  e <<f H _###?? F 
X
X 
X 	
X 
X 
X 
X 
X  	s   "% CBCCc           
        d }d}t          |	|
          r|                    |	d           d}ng }	|                     |||
          }|r7|j        j        }|                    |           |                    |          }|r^t          ||||||| j        |f|	||d|}|r>|d         }t          dt          |                    D ]}t          |||                    ng }|                    |j                            ||g          d	          5 }t          j        ||f||r|nd d
| d d d            n# 1 swxY w Y   |r|d         }|                    |           |rd|i}|r
|j        |d<   |gS g S )NFTr2   rB   )r*   )r`   compressionra   r   r;   rE   )r  rF   r0  rD   )ru   rL   r  rD   r*   updatereplace_schema_metadatarp   rf  rN   r-   rW   rT   rU   rX   rY   rZ   )r  r7   r   r   r]   rh  ra   fmdr  r`   rD   headcustom_metadatarb   _metarC   t_mdrg   r   r   ds                         r#   write_partitionz"ArrowDatasetEngine.write_partition[  sF   " J// 	LLTL222!NNJ&&r.QW&XX 	8(#CJJ'''))3)77A 	.(* &' /   G  :
q#g,,// : :A&ugaj9999GdH%566==  !,2A'Kwwt	 
                   .
##H--- 	A 'h(3JIs   D;;D?D?c                *   |d         d                              dd           }d |D             }|r`|s|j                            |dg          }t          t          j                  j        fd|                                D             }	|                    |d          5 }
t	          j        ||
fi |	 d d d            n# 1 swxY w Y   |j                            |dg          }|r||}d}n|d         d         d         }d	}t          |t          |                    D ]$}t          |||         d         d                    %|                    |d          5 }
|                    |
           d d d            d S # 1 swxY w Y   d S d S )
Nr   rD   c                .    g | ]}|d          d         |S )r   r0  r.   r5   ps     r#   r8   z5ArrowDatasetEngine.write_metadata.<locals>.<listcomp>  s%    >>>qQqT&\%=%=%=%=r%   _common_metadatac                $    i | ]\  }}|v 	||S r.   r.   )r5   kr   keywordss      r#   r   z5ArrowDatasetEngine.write_metadata.<locals>.<dictcomp>  s$    PPP1!x--q!---r%   rE   rR  r0  r;   )r   rT   rU   r   rX   write_metadataargsre  rW   rf  rN   r-   write_metadata_file)r  r1  r0  r   r   r   rb   rD   common_metadata_pathkwargs_metar   metadata_pathr  i_startr   r  s                  @r#   r  z!ArrowDatasetEngine.write_metadata  s>   q!400>>E>>> 	/ B')v{{D:L3M'N'N$%b&788=PPPPPPPWW1488 BC%fcAA[AAAB B B B B B B B B B B B B B B FKK{(;<<M $*aF+7CJJ// ? ?"5%(1+f*=>>>>-- /))#.../ / / / / / / / / / / / / / / / / /'	/ 	/s$   %CC	C	#FF
F
c           
     >   d}d}|                     di           }d|vrd|d<   d|vrt          j                    |d<   t          d'i |}d}t	          |          dk    r|                    |d                   rt          ||          \  }}}|j                            ||d         g          }|j                            |d	g          }|
s9|	                    |          r$t          j
        |fd
t          |          i|}d}nщrKt	          |          }fd|                    |          D             }|r|g k    rt          d d          nt	          |          dk    rpt          ||          \  }}}|j                            |d	g          }d	|v r<|
s#t          j
        |fd
t          |          i|}d}|                    d	           |}|!t          j        |fd
t          |          i|}	 t!          |                                          }|j        }n# t&          $ r d}|j        }Y nw xY w|dk    r7|r3|d}n0t+          d |j        D             |t/          |	                    }nd}|dk    r|rd}nd}g g }}|j        r|j        j        rt3          |j        j        j                  }t7          |          D ]r\  }}|j        j        r|j        j        |         nd}|                    t=          ||t?          j         g d          n|!                                                     stE          |	|          }i d|d|d|d|j        d|d|d|d|d|d|d|d|d |	d!|d"|d#|d$|d%|tG                      d&|iS )(zpyarrow.dataset version of _collect_dataset_info
        Use pyarrow.dataset API to construct a dictionary of all
        general information needed to read the dataset.
        Nr
   r   hiver   Fr;   r   rR  r   Tc                >    g | ]}|                               |S r.   )endswith)r5   r   r-  s     r#   r8   z<ArrowDatasetEngine._collect_dataset_info.<locals>.<listcomp>  s<       }}%;<<  r%   zLNo files satisfy the `parquet_file_extension` criteria (files must end with z).rO  c                    g | ]	}|j         
S r.   )total_byte_size)r5   rgs     r#   r8   z<ArrowDatasetEngine._collect_dataset_info.<locals>.<listcomp>R  s    KKK+KKKr%   r  objectr>  rp  physical_schemahas_metadata_filerD   r   valid_pathsr(  r&  rt   r   r  r)  r*  r  rC  r9  r,  rb   )r
   convert_stringr.   )$r   r   ParquetFileFormatr   rN   isdirr   rT   rU   existsparquet_datasetr$   findrO   rP   r
   nextget_fragmentsr  StopIterationrD   r   r   r   r   rK   rJ   r   dictionariesr   rw   rz   r?  	to_pandasr   r   )r  r  r   r&  rt   r(  r   r  r)  r*  r+  r,  r-  rb   rp  r  _dataset_kwargs_processed_dataset_kwargsr  basefns	meta_pathlen0	file_fragr  partition_objr9  r   r@   
dictionaryr  s               `                  r#   r#  z(ArrowDatasetEngine._collect_dataset_info  sd   .  !**Y3300.4ON+?**(-(?(A(AOH%$3$F$Fo$F$F! "u::??rxxa11?  7ubAAE4FKKs1v//EUK$899I' BIIi,@,@ * *2 0 
 %)!!'  5zz    "  
  ERKK$K0FK K K  
 ZZ!^^6ubAAE4T;$788Ic!! , -.! #.r?? 4 B
 )-% 

;'''! : &r?? , B	(R--//00I'7OO 	( 	( 	(I iOOO	(J w&& )$',$$'>KKi6JKKK!_--( ($$ $) z)) )#'  #(  *,R? 	r5 	"2?#9#?@@O$_55  4 3BO033 
 $$   *1 Ib9999!+!5!5!7!7 	 	 	 	 3?OTT
"

  !2
 bi	

 "
 ;
  !2
 *
 U
 w
  0
 
 
  !2
 -
  !
" !"4#
$ *"9";";  %
 
 	
s   =(H& &H<;H<c                @   |d         }|d         }|d         }|d         }|d         }|                     d|          j        d}t          |          pi }|r@|>g }|d         D ]3}	|	d	         d
k    r%|	d         |vr|                    |	d                    4|d                              di                                           }
|d         d         }|d         d         }|                     |                                ||
||          t          j        j                  }t          j	                  }|r|dgk    r
                    d           |!|r|dgk    s|                     dd          r|}|pd}|r|dgk    r                    |d           |refd|D             }|sg }g |d<   i |d<   ||d<   nCt          |          t          |          k    r#t          d                    |                    t          |||z   ||          \  }}||z   }|rpt!          |                              |          s0t          d                    |t          |                              t%          fd|D             |          |r|D ]}t          |j                  st)          |t                    r9|j        |d         k    r(t-          j        g |j        |d                   _        e|j        j        j        k    r,t-          j        g |j        j        j                  _        |j        j	        v r<t-          j        t-          j        |j        g           j                  |j        <   ||d<   ||d<   ||d<   S ) z|Use parquet schema and hive-partition information
        (stored in dataset_info) to construct DataFrame metadata.
        rD   rt   r&  rC  r9  r  Nr9   rW  rX  r@   rb   arrow_to_pandasr  r  )r  r  r  Tr2   index_columnsr.   c                    g | ]}|v|	S r.   r.   )r5   r  physical_column_namess     r#   r8   z6ArrowDatasetEngine._create_dd_meta.<locals>.<listcomp>  s$    SSSA=R4R4R14R4R4Rr%   rc   zNo partition-columns should be written in the 
file unless they are ALL written in the file.
physical columns: {} | partitions: {}zAcategories not in available columns.
categories: {} | columns: {}c                0    g | ]}|j         j        v|S r.   )rt   rJ   )r5   rY  r0  s     r#   r8   z6ArrowDatasetEngine._create_dd_meta.<locals>.<listcomp>  s(    IIIAq
8H/H/Ha/H/H/Hr%   )colsr  r   )r&  r@   r6  r8  r`   )r   rJ   r   r   ry   r=  empty_tablerK   rt   r9   rI   rL   rN   rO   r   r   rr   rs   r   ri   r"   r@   rz   CategoricalIndexr?  r@  )r  r/  rD   rt   r&  r  rC  r9   rr  r6   r  r  r  index_namescolumn_namesr`   _partitionsall_columnsrL  r0  r  s                      @@r#   r$  z"ArrowDatasetEngine._create_dd_meta  st    h'W%!,/
$\2!"34
 , 0 01BF K K Q /v66<" 	7!
*95 7 7CM*m;;F:55"))#f+666 'x0445FKKPPRR%h/0@A$X.?))  +') * 
 
 4:+,,DL)) 	+;4&00T***
 M  v%%"&&== &  E [b
 	5*..NN:tN444  	SSSSjSSSK 
-/\*13-.2<.//[!!S__44 <<BF-z= =   %=\J.{%
 %
!k "L0 	z??//<<  3396*d;FWFW3X3X   *IIIIIII+  D  	*  	9>** eT** y~q/I/I!#!4y~E!H" " "DJJ ^tz66!#!4y~DJO" " "DJJ ^t|33+-9).LLL"j, , ,D( !&W%/\"%/\"r%   c                  %& |d         }|d         %|d         }|d         }|d         }|d         }|d         }|d         }|d	         }	|d
         }
|d         }|d         }|d         }|d         &|d         }t          |d         %          }t                      }|kt          |t                    D ]T}|\  }}}|dk    r1t	          |t          t          t
          f          st          d          |                    |           Ui }|rt          |          dk    r|ng }t          |	j
                  D ]\  }}||v s||v r
||
v r|||<   t          |||||t          |                    }||||	d|}|du r,|s*|s(d t          |j        t                    D             g |fS d}|t          |          }%|||||	|||||d         d}|s|dk    s|t          |j                  k    rJt          d |                    |          D             d           }|                     ||          \  }}n|r2|r0t          d |                    |          D             d            }n,t          |j        t                    }&r%&fd!|D             }g g }}|ri } d"t%          ||          z   }g }!t          t'          dt          |          |                    D ]:\  }"}#|!                    ||"f           | j        ||#|#|z            |f| |!d#         <   ;d$ }$|$|!f| d%|z   <   t+          d%|z   |                                           \  }}|||fS )&aI  pyarrow.dataset version of _construct_collection_plan
        Use dataset_info to construct the general plan for
        generating the output DataFrame collection.

        The "plan" is essentially a list (called `parts`) of
        information that is needed to produce each output partition.
        After this function is returned, the information in each
        element of `parts` will be used to produce a single Dask-
        DataFrame partition (unless some elements of `parts`
        are aggregated together in a follow-up step).

        This method also returns ``stats`` (which is a list of
        parquet-metadata statistics for each element of parts),
        and ``common_metadata`` (which is a dictionary of kwargs
        that should be passed to the ``read_partition`` call for
        every output partition).
        rp  r   r   r  r(  r)  r  r`   rD   r9  rC  r&  r  r  rb   r,  Nr   r   z2Value of 'in' filter must be a list, set or tuple.r;   )rC  r&  r   rD   Fc                    g | ]	}d |ddfi
S rI  Nr.   )r5   rn   s     r#   r8   zAArrowDatasetEngine._construct_collection_plan.<locals>.<listcomp>q  s4       ! y$56  r%   rT  r
   )r   r  r(  r   
ds_filtersrD   stat_col_indicesr  r)  rC  r  r   c              3     K   | ]}|V  d S r   r.   r5   frags     r#   r   z@ArrowDatasetEngine._construct_collection_plan.<locals>.<genexpr>  s"      ??$??????r%   c                *    t          | j                  S r   r    r   xs    r#   <lambda>z?ArrowDatasetEngine._construct_collection_plan.<locals>.<lambda>  s    .qv66 r%   c              3     K   | ]}|V  d S r   r.   r  s     r#   r   z@ArrowDatasetEngine._construct_collection_plan.<locals>.<genexpr>  s"      CCdTCCCCCCr%   c                *    t          | j                  S r   r  r  s    r#   r  z?ArrowDatasetEngine._construct_collection_plan.<locals>.<lambda>  s    "216":": r%   c                X    g | ]&}|                     j                  d          v $|'S )rG   )splitrT   )r5   filefr   r  s     r#   r8   zAArrowDatasetEngine._construct_collection_plan.<locals>.<listcomp>  sA     ! ! !! ;;rv..r2kAA AAAr%   zgather-pq-parts-rG   c                :    g g }}| D ]\  }}||z  }|r||z  }||fS r   r.   )parts_and_statsr1  r   partstats        r#   _combine_partszEArrowDatasetEngine._construct_collection_plan.<locals>._combine_parts  sC    #%r5E&5 * *
d *!TME %<'r%   zfinal-)r   rr   r   rK   r"   rS   	TypeErroraddrN   r   rJ   r   ra  r_  r    r   r  _collect_file_partsr   rf  r   r   compute)'r  r/  rp  r   r  r(  r)  r  r`   rD   r9  rC  r&  r  rb   r,  filter_columnsfilterr6   r   rA   r  _index_colsr   r@   r  r  dataset_info_kwargs
file_fragsr1  r   	all_filesgather_parts_dskfinalize_listtask_ifile_ir  r   r  s'                                        @@r#   r%  z-ArrowDatasetEngine._construct_collection_plan  s   * $$y)'(:;()<= -	()<=!,/
h'&'89!,/
!,/
()<="=1h' 5-.
 

 !'T::: ( (%R::jsD%6H&I&I:#L   ""3''''
 %6X3z??a;O;OjjVX .. 	+ 	+GAt{""dn&<&<?**)* & 3 !!
 
 %$	
 

 
 %%/?%7% %+BH:J%K%K%K     
/88J  0!2$ 0!2"$%i0
 
  ?	T!Q&&!CMM11  ??""2"2:">">???66  J 22:?RSSLE55
  : "CCb&6&6z&B&BCCC::  		 #281ABBB	 ! ! ! ! !%.! ! !I r5E T#% )HY@S,T,TT "&/!S^^-?@@' '  NFF "(($888/!&64F+F"FG+;$]2%677( ( ( 6D]4S D1&x$8HIIQQSSue]**r%   c                  " |d         }|d         }|d         }|d         }|d         }t          |t                    s|g}n|sg g fS t          |d         t                    ra|s|s|du rd |D             d fS t          t          j        |fd	t          |          it          d i |                                          }n|}|d
         }	|d         }
|d         }|d         }|d         }|d         }t          t                    }t          t                    }t          t                    }t          |          dk    }i }i }|D ]k}|j
        }t          j        |j                  ""fd|D             ||<   |                    |
|          D ]}|j        }|s|r.||                                 |j        }t!          |          s;nd g||<   C|D ]}||                             |j                   |rt'          |t          |                    }|r||j        |j        g d}n|j        |j        d}g }|                                D ]}||v r||         d         }||         d         }||         d         } t          |t.                    rt1          j        |          n|}t          |t.                    rt1          j        |          n|}|                    |d           }!|	s|r|du s|s||!r||!k     rd}i }i } n[|r!|d                             |||| d           n|||| gz  }|||<   |r|d                             d|i           |g dz  }|rE||                             |           |s(||                             t7          |                     ِ m|sg g fS t9          |||||||| j        |||dd	  	        S )!Nr   r  r(  rC  r  r   Fc                    g | ]	}d |ddfi
S r  r.   )r5   file_or_frags     r#   r8   z:ArrowDatasetEngine._collect_file_parts.<locals>.<listcomp>  s4       $ |T489  r%   r   r   r  rD   r  r  r)  r;   c                8    g | ]}|j         |j                  fS r.   r@   r5   	hive_partraw_keyss     r#   r8   z:ArrowDatasetEngine._collect_file_parts.<locals>.<listcomp>  s4     * * *?H).!9:* * *r%   )rD   )file_path_0num-rowsr  r9   )r  r  r   r   r   Tr9   )r@   r   r   r   r@   )NNN )r   rc   r  	data_path)make_part_kwargsr.   )r"   rK   r)   r   r
   r$   r   r  r   intr   _get_partition_keysr   split_by_row_groupr   ensure_complete_metadatarN   r   idr   num_rowsr  ri   r   rz   	Timestampr   rS   r   
_make_part)#r  files_or_fragsr  r   r  r(  rC  r  r  r   r  rD   r  r  r)  file_row_groupsfile_row_group_statsfile_row_group_column_statssingle_rg_partshive_partition_keys	cmax_lastr  fpathr  row_group_infor   r   scstatsr@   cmincmaxr   lastr  s#                                     @r#   r  z&ArrowDatasetEngine._collect_file_parts  s    !&./AB/0CD(6
-.?@ .$// 	,-NN 	r6M nQ'-- 	( % 
 8IU8R8R (6     " *2 &8888   -// JJ (J &i0(6
$X../AB/0CD'4	 &d++*400&1$&7&7#.//14 	# b	Y b	YINE 01OPPH* * * *LV* * *& "44Z4OO XY XY!%$ (8  &-55777)-~.. ! ! /3VOE*!/ JY JYI#E*11),???( HY%7%t,<'='=& &
 + /4,5,>3<3L+-	! !AA -6,>3<3L! !A "$$4$9$9$;$; 3A 3AD#z11'1$'7'>'1$'7'>-7-=l-K
 (2$'A'A%.BL$6$6$6)- !% (2$'A'A%.BL$6$6$6)- !%
 (1}}T4'@'@$+!.(1!.6F$6N6N'8 7O
 (,|| =B(9?A(<FH(C(-#2 
!G$%iL$7$7483737:D	)* )*%& %& %& %& %+tT:.F$FF26	$#2 !A$%iL$7$7$G$G$G$G$*.@.@.@$@FF, Y07>>qAAA#2 Y ;E B I I%PV-- X X XUJYXYv  	r6M $ 'N"5!+	 
 
 
 	
r%   c                    |j                             d ||fD                       }|                    |d          }|r|dS d|||fiS )z1Generate a partition-specific element of `parts`.c                    g | ]
}|d k    |S )r   r.   r  s     r#   r8   z1ArrowDatasetEngine._make_part.<locals>.<listcomp>  s     M M MqQ"WWWWWr%   NrI  )rT   rU   r   )	r  r]   rg_listr   rc   r  r  rn   pkeyss	            r#   r
  zArrowDatasetEngine._make_part  sc     FKK M MY,A M M MNN	""9d33 	U]4)We455r%   c	                   t          |t          j                  r|}
n$d}
|	                    di                               dd          }|r|du p|ot          |t          t
          f           }|st          ||          rt          j        |fdt          |          it          di |	                    di           }t          |
                                          }t          |          dk    sJ |dgk    rt          |d         |          n|d         }
t          j        |
j                  fd|D             }|
rhg }|D ]8}|d|j        v r|                    d           #|                    |           9|
                    d	|||rt%          |          nd
          }nt'          ||||||fi |	}|r.t          |t
                    rd |D             }|D ]}|j        |j        j        vr|                    |j        d          }t          |j                  s5t/          j        t3          j        t          |          |                    }nst3          j        t          |          |j                            |          d          }t.          j                            |t/          j        |j                            }|                    |j        |          }
|S )zRead in a pyarrow tableNr
   r   r   r;   r   c                8    g | ]}|j         |j                  fS r.   r  r  s     r#   r8   z2ArrowDatasetEngine._read_table.<locals>.<listcomp>  s7     " " "! ^Xin%=>" " "r%   __index_level_0__F)r   rD   r9   r  c                    i | ]\  }}||	S r.   r.   )r5   r  r   s      r#   r   z2ArrowDatasetEngine._read_table.<locals>.<dictcomp>  s    ;;;&1aA;;;r%   i4r  r.   )r"   r   ParquetFileFragmentr   r)   rK   r   r
   r$   r   r  rN   r   r  r   rJ   r   to_tabler   r   r@   rD   ri   r;  arrayr   fullget_locDictionaryArrayfrom_arraysappend_column)r  rJ  r   r   r9   rD   r   rC  rc   rb   r  r   missing_partitioning_inforp  fragsr  r@   rK  	keys_dictrL  catarrcat_indr  s                          @r#   r:  zArrowDatasetEngine._read_table  sk    lE$=>> ,	DD D "::i4488NNL 6$ 6 P NL3+)N)N%N	 & ) OG^,T,T ]  *2 &BB

9b(A(ABB 
 R--//005zzQ "dV++ !q:666q  !4T5NOO" " " "%/" " "
  	D & &<*fl::':;;;KK%%%%--!:AK-g666t	 (  KK 0   K  	Q*Z66 	Q;;N;;;I' Q Q	>);)AAA $--	==Cy~..  hrws;/?/?'E'EFF"$',,in.D.DS.I.IQU# # # !0<<#RXin%=%=  #.";";INC"P"PKr%   r  r  c                  |                     di                                d          }g d }|                    |           |r?                    t          j                    t	          j        d          ij                    |dk    r                     t          j                    n|dk    r                    |           fd}t                    dk    r|S d S )Nr  types_mapperc                    | t          j                    k    rt          j        d          S t          j        |           S )Nr   )r;  stringrz   StringDtype
ArrowDtype)pyarrow_dtypes    r#   pyarrow_type_mapperzFArrowDatasetEngine._determine_type_mapper.<locals>.pyarrow_type_mapper  s5     	++~i000}]333r%   r   r  c                4    D ]} ||           }||c S dS )zBTry all type mappers in order, starting from the user type mapper.Nr.   )r7  type_converterconverted_typetype_mapperss      r#   default_types_mapperzGArrowDatasetEngine._determine_type_mapper.<locals>.default_types_mapper!  sB    ". * *!/!>!>!-)))) .* *r%   r   )r   r   r;  r4  rz   r5  PYARROW_NULLABLE_DTYPE_MAPPINGrN   )r  r  r  rb   user_mapperr8  r=  r<  s          @r#   _determine_type_mapperz)ArrowDatasetEngine._determine_type_mapper  s    jj!2B77;;NKK	4 	4 	4 ",,,  	NbnY.G.G H LMMM ,,, > BCCCCi'' 3444	* 	* 	* 	* 	* |q  '' ! r%   rK  c                v   |                     di           }|                    ddd            | j        d||d|}|||d<    |j        dd|i|}|rt	          |j        t          j                  rt	          |j        t          j                  st          j	        j
                            |j        j                  rv|j        j        t          j        d          t          j        t          j                              fvr1|j                            t          j        d                    |_        |S )	Nr  F)r   ignore_metadatar0  r2  r&  r   r.   )r   r  r@  r  r"   rt   rz   r{   
MultiIndexapitypesis_string_dtyper>  r5  r6  r;  r4  astype)	r  rK  r&  r  r  rb   _kwargsr2  ress	            r#   r=  z)ArrowDatasetEngine._arrow_table_to_pandas+  sJ    **.33uGGHHH1s1 
')
 
 
 

 #&2GN##k#EEzEWEE 	D39bh//	D sy"-88	D ,,SY_==		D
 	N9--r}RY[[/I/IJK K 	((	)B)BCCCI
r%   c                    |                     |d          5 }t          j        |          j        }d d d            n# 1 swxY w Y   |r|                    |           |S )Nr  )rW   rX   r   r*   rZ   )r  r   r   	file_pathro   r0  s         r#   collect_file_metadataz(ArrowDatasetEngine.collect_file_metadataL  s    WWT4   	.A>!$$-D	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	*y)))s   =AAc                &   d }|D ]}|rt          ||           |}|rr|j                            |dg          }|                    |d          5 }|st	          d          |                    |           d d d            n# 1 swxY w Y   d S |S )NrR  rE   zCannot write empty metdata!)r-   rT   rU   rW   rO   r  )r  	meta_listr   out_pathr0  r  r  r   s           r#   aggregate_metadataz%ArrowDatasetEngine.aggregate_metadataT  s     	 	E "4//// 	FKK;(?@@M-- . D$%BCCC((---. . . . . . . . . . . . . . . 4Ks   'BBB)NNNNNNr  NNFr   N)Nr.   r.   NN)FNFNrO  N)FN)r7   rw  rx  ry  )NNNNFN)F)NNNN)NF)rK  ry  rx  rw  )r   r   r   classmethodr  r2  r4  rN  rv  r  r  r  r#  r$  r%  r  r
  r:  r@  r=  rL  rP  r.   r%   r#   r   r     s       
 =
 =
 [=
~ 
  #"#J; J; J; [J;X ) ) [)  { { { [{z  ~Q ~Q ~Q [~Q@ <@    [>  F F F [FP / / / [/: R
 R
 [R
h ~ ~ [~@ v+ v+ [v+p o
 o
 [o
b 
 6 6 6 [6& j j [jX "5$( $( $( $( [$(L 
     [@   [   [  r%   r   )r.   Tr   )FT)a
__future__r   r   r   r  r  collectionsr   r   	functoolsr   numpyr   r  rz   r   r;  pyarrow.parquetrQ  rX   fsspec.corer   r   fsspec.implementations.arrowr	   r
   r   r   r  r!  	dask.baser   	dask.corer   dask.dataframe.backendsr   dask.dataframe.io.parquet.utilsr   r   r   r   r   r   r   r   r   dask.dataframe.io.utilsr   r   r   dask.dataframe.utilsr   r   dask.delayedr   
dask.utilsr   r    int8	Int8Dtypeint16
Int16Dtypeint32
Int32Dtypeint64
Int64Dtypeuint8
UInt8Dtypeuint16UInt16Dtypeuint32UInt32Dtypeuint64UInt64Dtypebool_BooleanDtyper4  r5  float32Float32Dtypefloat64Float64Dtyper>  r$   r-   rp   ru   rw   r   r   r   r   r   r?   r   r   r   r.   r%   r#   <module>rw     sX   " " " " " "     # # # # # #                               ? > > > > > > > 7 7 7 7 7 7 $ $ $ $ $ $                    ; ; ; ; ; ;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 Y X X X X X X X X X P P P P P P P P             3 3 3 3 3 3 3 3 BGII|r|~~BHJJBHJJBHJJBHJJBIKK!!BIKK!!BIKK!!BHJJ!!BIKK!!BJLL/"/##BJLL/"/##" *; ; ;
  6 E E E EP
 
 
= = = = = = = =    = = =@" " "J0 0 0"     D5 D5 D5 D5X~ ~ ~ ~ ~ ~ ~ ~ ~ ~r%   