>ieiddlmZddlZddlmZmZddlmZddlZddl Z ddl m Z ddl mZ ddlmZn #e$rdZYnwxYwdZdZdd Zd Zd Zd Z dd ZeGddeZeGddeZdS)) annotationsN)Protocolruntime_checkable)uuid4)LocalFileSystem)parsecN|o#t|tpt|S)z'Check if an fsspec file-system is local) isinstancer_is_local_fs_pyarrowfss 7lib/python3.11/site-packages/dask/dataframe/io/utils.py _is_local_fsrs,  2'' $  # #c|r?t|drt|jSt|dr |jdkSdS)z-Check if a pyarrow-based file-system is localr type_namelocalF)hasattrr r rr s rr r sP + 2t   +'.. . R % % +<7* * 5rc|dkrddlm}|j}nd}|jduod|jv}|rst j|jdd}d|d gD}d |d gD}ni}i} tt|D]} || } | j |vrV| j |vr?tj gd j || j j} n0|| j } n" || j} n#t $rYwxYw| | | j <|r |D]} d | | <| S)z4Convert a pyarrow.Schema object to pandas dtype dictnumpy_nullabler)PYARROW_NULLABLE_DTYPE_MAPPINGc*|SN)to_pandas_dtype)ts rz%_get_pyarrow_dtypes...s 1 1 3 3rNspandasutf8c pi|]3}|d|dd|d4S) field_namenameN numpy_typeget.0cs r z'_get_pyarrow_dtypes..4sJ" " "  EE,fd 3 3 4 4a o" " " rcolumnsc i|]Y}|ddv |d|d|dd|dddZS) pandas_type)datetime datetimetzmetadatarr Ntimezoner"r$s rr'z'_get_pyarrow_dtypes..8sw   #===!J-= EE,fd 3 3 4 4a m6G6GD77>==rzM8[ns]dtypecategory)dask.dataframe.io.parquet.arrowrr#r-jsonloadsdecoderangelenr pdSeriesdt tz_localizer0typeNotImplementedError)schema categories dtype_backendr type_mapperhas_pandas_metadatapandas_metadatapandas_metadata_dtypestzdtypesifield numpy_dtypecats r_get_pyarrow_dtypesrK's(((RRRRRR48 33 !/5V)v:V$*V_Y%?%F%Fv%N%NOO" " $((B77" " "   %((B77    "$ F 3v;;  ))q  :/ / /zRIb1114@@EJPPV 5UZ@  )k%*55 &    )uz% % %C$F3KK Ms)D:: EEcffd|Dfd|pgD}t|dkrd}nVt|dkr"|d}|ddkr |d|_n!tj||}tj|}|r ||j_|S) aNGet the final metadata for the dask.dataframe Parameters ---------- to_read_columns : list All the columns to end up with, including index names file_dtypes : dict Mapping from column name to dtype for every element of ``to_read_columns`` index_cols : list Subset of ``to_read_columns`` that should move to the index column_index_names : list The values for df.columns.name for a MultiIndex in the columns, or df.index.name for a regular Index in the columns Returns ------- meta : DataFrame c fi|]-}|tjg|d.S)int64r/)r8r9r#)r%r& file_dtypess rr'z%_meta_from_dtypes..rsE   @A29R{q':: ; ; ;   rc:g|]}|S)pop)r%r&datas r z%_meta_from_dtypes..us#555qtxx{{555rrN__index_level_0__)names)index)r7r r8 MultiIndex from_arrays DataFramer(rW)to_read_columnsrO index_colscolumn_index_namesindexesrXdfrSs ` @r_meta_from_dtypesra]s*    ET   D6555J$4"555G 7||q ZA    a=/ / /#AEJ ))')DD d% ( ( (B.-  Irc(tjS)z0Simple utility function to get random hex string)rhexrQrr_guidrds 77;rc4||S||S)z5Helper function to place an object on a context stack) enter_context)objstacks r _set_contextris  }   s # ##rc `fd|DS|pi}|dd}|dkrƉtsttjtdkr|ddpdgt|z}dd}|dkrtd |d fd t||DSfd |DSfd |DS)aReturn a list of open-file objects given a list of input-file paths. WARNING: This utility is experimental, and is meant for internal ``dask.dataframe`` use only. Parameters ---------- paths : list(str) Remote or local path of the parquet file fs : fsspec object, optional File-system instance to use for file handling context_stack : contextlib.ExitStack, Optional Context manager to use for open files. open_file_func : callable, optional Callable function to use for file opening. If this argument is specified, ``open_file_func(path, **kwargs)`` will be used to open each file in ``paths``. Default is ``fs.open``. precache_options : dict, optional Dictionary of key-word arguments to use for precaching. If ``precache_options`` contains ``{"method": "parquet"}``, ``fsspec.parquet.open_parquet_file`` will be used for remote storage. **kwargs : Key-word arguments to pass to the appropriate open function Nc >g|]}t|fiSrQ)ri)r%path context_stackkwargsopen_file_funcs rrTz%_open_input_files..sE    7777 G G   rmethodparquetz 2021.11.0 row_groups cache_typepartsz<'parts' `cache_type` required for 'parquet' precaching, got .c \g|](\}}ttj|f|d)S))r rr)rifsspec_parquetopen_parquet_file)r%rlrgsrmr rns rrTz%_open_input_files..sk    c 0"        rc Hg|]}tj|fiSrQriopen)r%rlrmr rns rrTz%_open_input_files..s7WWW WRWT44V44mDDWWWrc Fg|]}tt|fiSrQr{)r%rlrmrns rrTz%_open_input_files..s3 P P P$Ld--f--} = = P P Pr) copyrRr parse_versionfsspec __version__updater7 ValueErrorzip) pathsr rmroprecache_optionsrnprecacherrrss ``` ` r_open_input_filesrsF!          ).B4466##Hd33HI NR    &, - - k0J0J J J &'''ZZ d33LU8K ZZ g66  &"&&&        ! 33     WWWWWWQVWWWW P P P P P% P P PPrc4eZdZdZedZdZdZdS)DataFrameIOFunctionzkDataFrame IO function with projectable columns Enables column projection in ``DataFrameIOLayer``. ct)z$Return the current column projectionr=selfs rr(zDataFrameIOFunction.columnss "!rct)zUReturn a new DataFrameIOFunction object with a new column projection r)rr(s rproject_columnsz#DataFrameIOFunction.project_columnss "!rct)z Return a new DataFrame partitionr)rargsrns r__call__zDataFrameIOFunction.__call__s!!rN)__name__ __module__ __qualname____doc__propertyr(rrrQrrrrsW ""X"""" """""rrceZdZddZddZdS) SupportsLockreturnobjectcdSrrQrs racquirezSupportsLock.acquire rcdSrrQrs rreleasezSupportsLock.releaserrN)rr)rrrrrrQrrrrs<          rrr)NNNN) __future__rr3typingrruuidrrpandasr8fsspec.implementations.localrpackaging.versionrrfsspec.parquetrqrw ImportErrorrr rKrardrirrrrQrrrs"""""" ........ 888888444444+++++++NNN   3333l'''T $$$  KQKQKQKQ\"""""(""",     8     s 5??