
    'e8                         d dl Z d dlZd dlZd dlZd dlZd dlmZ d dl	Z
d dlmZmZmZmZ  ej        ddd          Z G d d          Z G d	 d
e          Z G d de          Z G d de          Zd Zd ZddZdS )    N)join)doc_stringify_path_is_path_like	_DEPR_MSGzfilesystem.LocalFileSystemz2.0.0zfs.LocalFileSystemc                       e Zd ZdZd Zd ZddZd Zd Zd Z	dd	Z
d
 Zd ZddZd Zd Zd Zd Z	 	 ddZddZed             ZdS )
FileSystemz(
    Abstract filesystem interface.
    c                     |                      |d          5 }|                                cddd           S # 1 swxY w Y   dS )z
        Return contents of file as a bytes object.

        Parameters
        ----------
        path : str
            File path to read content from.

        Returns
        -------
        contents : bytes
        rbN)openread)selfpathfs      2  /croot/pyarrow_1707330824290/_h_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold/lib/python3.11/site-packages/pyarrow/filesystem.pycatzFileSystem.cat)   s     YYtT"" 	a6688	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	s   8<<c                     t           )z
        Return list of file paths.

        Parameters
        ----------
        path : str
            Directory to list contents from.
        NotImplementedErrorr   r   s     r   lszFileSystem.ls9   
     "!    Fc                     t           )z
        Delete the indicated file or directory.

        Parameters
        ----------
        path : str
            Path to delete.
        recursive : bool, default False
            If True, also delete child paths for directories.
        r   r   r   	recursives      r   deletezFileSystem.deleteD   
     "!r   c                 $   t          |          }|                     |          }|d         dk    r|d         S d}|                     |          D ]?\  }}}|D ]6}|                     ||          }||                     |          d         z  }7@|S )z
        Compute bytes used by all contents under indicated path in file tree.

        Parameters
        ----------
        path : str
            Can be a file path or directory.

        Returns
        -------
        usage : int
        kindfilesizer   )r   statwalk
_path_join)	r   r   	path_infototalrootdirectoriesfiles
child_pathabspaths	            r   
disk_usagezFileSystem.disk_usageQ   s     t$$IIdOO	V&&V$$(,		$ 	4 	4$D+u# 4 4
//$
;;7++F334 r   c                 6    | j                             |          S N)pathsepr   )r   argss     r   r%   zFileSystem._path_joink   s    |  &&&r   c                      t          d          )zl
        Information about a filesystem entry.

        Returns
        -------
        stat : dict
        zFileSystem.statr   r   s     r   r#   zFileSystem.statn   s     ""3444r   c                 0    |                      ||          S )z.
        Alias for FileSystem.delete.
        r   )r   r   s      r   rmzFileSystem.rmx   s     {{49{555r   c                 .    |                      ||          S )z.
        Alias for FileSystem.rename.
        )renamer   r   new_paths      r   mvzFileSystem.mv~   s     {{4***r   c                      t          d          )z
        Rename file, like UNIX mv command.

        Parameters
        ----------
        path : str
            Path to alter.
        new_path : str
            Path to move to.
        zFileSystem.renamer   r8   s      r   r7   zFileSystem.rename   s     ""5666r   Tc                     t           )z
        Create a directory.

        Parameters
        ----------
        path : str
            Path to the directory.
        create_parents : bool, default True
            If the parent directories don't exists create them as well.
        r   r   r   create_parentss      r   mkdirzFileSystem.mkdir   r   r   c                     t           )z
        Return True if path exists.

        Parameters
        ----------
        path : str
            Path to check.
        r   r   s     r   existszFileSystem.exists   r   r   c                     t           )z
        Return True if path is a directory.

        Parameters
        ----------
        path : str
            Path to check.
        r   r   s     r   isdirzFileSystem.isdir   r   r   c                     t           )z
        Return True if path is a file.

        Parameters
        ----------
        path : str
            Path to check.
        r   r   s     r   isfilezFileSystem.isfile   r   r   c                     t           )zf
        Returns True if this FileSystem is a unix-style file store with
        directories.
        r   r   s    r   _isfilestorezFileSystem._isfilestore   s
    
 "!r   Nc                 \    ddl m}  |||||           }|                    |||          S )a]  
        Read Parquet data from path in file system. Can read from a single file
        or a directory of files.

        Parameters
        ----------
        path : str
            Single file path or directory
        columns : List[str], optional
            Subset of columns to read.
        metadata : pyarrow.parquet.FileMetaData
            Known metadata to validate files against.
        schema : pyarrow.parquet.Schema
            Known schema to validate files against. Alternative to metadata
            argument.
        use_threads : bool, default True
            Perform multi-threaded column reads.
        use_pandas_metadata : bool, default False
            If True and file has custom pandas schema metadata, ensure that
            index columns are also loaded.

        Returns
        -------
        table : pyarrow.Table
        r   )ParquetDataset)schemametadata
filesystem)columnsuse_threadsuse_pandas_metadata)pyarrow.parquetrJ   r   )	r   r   rN   rL   rK   rO   rP   rJ   datasets	            r   read_parquetzFileSystem.read_parquet   sZ    6 	322222 .fx,02 2 2||G0C  E E 	Er   r   c                     t           )3
        Open file for reading or writing.
        r   r   r   modes      r   r   zFileSystem.open   s
     "!r   c                     dS )N/ rG   s    r   r0   zFileSystem.pathsep   s    sr   FT)NNNTFr   )__name__
__module____qualname____doc__r   r   r   r-   r%   r#   r5   r:   r7   r?   rA   rC   rE   rH   rS   r   propertyr0   rZ   r   r   r	   r	   $   sQ           	" 	" 	"" " " "  4' ' '5 5 56 6 6 6+ + +7 7 7" " " "	" 	" 	"	" 	" 	"	" 	" 	"" " " FJ;@E E E EB" " " "   X  r   r	   c                       e Zd ZdZ fdZed             Zed             Z ee	j
                  d             Z
 ee	j                  dd            Z ee	j                  d             Z ee	j                  d	             Z ee	j                  d
             Z ee	j                  d             Z ee	j                  dd            Zed             Zd Z xZS )LocalFileSystemNc                     t          j        t          t          d           t	                                                       d S N   
stacklevel)warningswarn_FS_DEPR_MSGFutureWarningsuper__init__)r   	__class__s    r   ro   zLocalFileSystem.__init__   s6    lMa@@@@r   c                     | j         Rt          j                    5  t          j        d           t	                      | _         d d d            n# 1 swxY w Y   | j         S )Nignore)	_instancerj   catch_warningssimplefilterrd   clss    r   _get_instancezLocalFileSystem._get_instance   s    = (** 2 2%h/// / 1 12 2 2 2 2 2 2 2 2 2 2 2 2 2 2 }s   (AAAc                 l    t          j        t          t          d           |                                 S rf   )rj   rk   rl   rm   rx   rv   s    r   get_instancezLocalFileSystem.get_instance  s+    lMa@@@@  """r   c                 |    t                    t          fdt          j                  D                       S )Nc              3   8   K   | ]}t          |          V  d S r/   )pjoin).0xr   s     r   	<genexpr>z%LocalFileSystem.ls.<locals>.<genexpr>
  s+      ??eD!nn??????r   )r   sortedoslistdirr   s    `r   r   zLocalFileSystem.ls  s=    t$$????bj.>.>??????r   Tc                 |    t          |          }|rt          j        |           d S t          j        |           d S r/   )r   r   makedirsr?   r=   s      r   r?   zLocalFileSystem.mkdir  s>    t$$ 	KHTNNNNNr   c                 ^    t          |          }t          j                            |          S r/   )r   r   r   rC   r   s     r   rC   zLocalFileSystem.isdir  s#    t$$w}}T"""r   c                 ^    t          |          }t          j                            |          S r/   )r   r   r   rE   r   s     r   rE   zLocalFileSystem.isfile  #    t$$w~~d###r   c                     dS )NTrZ   rG   s    r   rH   zLocalFileSystem._isfilestore  s    tr   c                 ^    t          |          }t          j                            |          S r/   )r   r   r   rA   r   s     r   rA   zLocalFileSystem.exists"  r   r   r   c                 B    t          |          }t          ||          S rU   )rW   )r   r   rV   s      r   r   zLocalFileSystem.open'  s$    
 t$$Dt$$$$r   c                 $    t           j        j        S r/   )r   r   seprG   s    r   r0   zLocalFileSystem.pathsep/  s    w{r   c                 H    t          |          }t          j        |          S )z8
        Directory tree generator, see os.walk.
        )r   r   r$   r   s     r   r$   zLocalFileSystem.walk3  s     t$$wt}}r   r\   r]   )r^   r_   r`   rs   ro   classmethodrx   rz   r   r	   r   r?   rC   rE   rH   rA   r   rb   r0   r$   __classcell__)rp   s   @r   rd   rd      s       I       [ # # [# 	S@ @ @ 	S	    	S	# # # 	S	$ $ $ 	S	 !!  "! 	S	$ $ $ 	S% % % %   X      r   rd   c                      e Zd ZdZd Z eej                  d             Z eej                  d             Z eej	                  d             Z	 eej
                  dd            Z
 eej                  d             Z eej                  dd
            Z eej                  dd            ZddZd ZdS )DaskFileSystemzJ
    Wraps s3fs Dask filesystem implementation like s3fs, gcsfs, etc.
    c                 L    t          j        dt          d           || _        d S )Nz~The pyarrow.filesystem.DaskFileSystem/S3FSWrapper are deprecated as of pyarrow 3.0.0, and will be removed in a future version.rg   rh   )rj   rk   rm   fs)r   r   s     r   ro   zDaskFileSystem.__init__@  s3    La	) 	) 	) 	) r   c                      t          d          NzUnsupported file system APIr   r   s     r   rC   zDaskFileSystem.isdirG      !"?@@@r   c                      t          d          r   r   r   s     r   rE   zDaskFileSystem.isfileK  r   r   c                     dS )zh
        Object Stores like S3 and GCSFS are based on key lookups, not true
        file-paths.
        FrZ   rG   s    r   rH   zDaskFileSystem._isfilestoreO  s	     ur   Fc                 X    t          |          }| j                            ||          S )Nr4   )r   r   r5   r   s      r   r   zDaskFileSystem.deleteW  s'    t$$wzz$)z444r   c                 T    t          |          }| j                            |          S r/   )r   r   rA   r   s     r   rA   zDaskFileSystem.exists\  s#    t$$w~~d###r   Tc                     t          |          }|r| j                            |          S | j                            |          S r/   )r   r   mkdirsr?   r=   s      r   r?   zDaskFileSystem.mkdira  s?    t$$ 	'7>>$'''7==&&&r   r   c                 X    t          |          }| j                            ||          S r   )r   r   r   rV   s      r   r   zDaskFileSystem.openi  s)    
 t$$w||Dt|,,,r   c                 X    t          |          }| j                            ||          S )N)detail)r   r   r   )r   r   r   s      r   r   zDaskFileSystem.lsq  s'    t$$wzz$vz...r   c                 T    t          |          }| j                            |          S )z9
        Directory tree generator, like os.walk.
        )r   r   r$   r   s     r   r$   zDaskFileSystem.walku  s%     t$$w||D!!!r   Nr[   r\   r]   )r^   r_   r`   ra   ro   r   r	   rC   rE   rH   r   rA   r?   r   r   r$   rZ   r   r   r   r   ;  s           	S	A A A 	S	A A A 	S	 !!  "! 	S	5 5 5 5 	S	$ $ $ 	S	' ' ' ' 	S- - - -/ / / /" " " " "r   r   c                   z    e Zd Z eej                  d             Z eej                  d             ZddZdS )S3FSWrapperc                     t          t          |                    }	 | j                            |          }t	          |          dk    r|d         |k    rdS dS # t
          $ r Y dS w xY w)N   r   FT_sanitize_s3r   r   r   lenOSErrorr   r   contentss      r   rC   zS3FSWrapper.isdir  sz    OD1122	wzz$''H8}}!!hqkT&9&9ut 	 	 	55	s   9A 
A)(A)c                     t          t          |                    }	 | j                            |          }t	          |          dk    o|d         |k    S # t
          $ r Y dS w xY w)Nr   r   Fr   r   s      r   rE   zS3FSWrapper.isfile  sp    OD1122	wzz$''Hx==A%=(1+*== 	 	 	55	s   8A 
A%$A%Fc              #   &  K   t          t          |                    }t                      t                      }t          | j                            ||                    D ]N}|d         }|d         dk    r                    |           ,|d         dk    r9|                    |           Ot          fd|D                       }t          d D                       ||fV  D ]}|                     ||          E d{V   dS )	z
        Directory tree generator, like os.walk.

        Generator version of what is in s3fs, which yields a flattened list of
        files.
        )refreshKeyStorageClass	DIRECTORYBUCKETc                 L    g | ] }|vt          j        |          d          !S r   	posixpathsplit)r~   r   r)   s     r   
<listcomp>z$S3FSWrapper.walk.<locals>.<listcomp>  s<     1 1 1!K// "**1-///r   c                 B    g | ]}t          j        |          d          S r   r   )r~   r   s     r   r   z$S3FSWrapper.walk.<locals>.<listcomp>  s7     4 4 4"# (oa003 4 4 4r   N)	r   r   setlistr   _lsaddr   r$   )r   r   r   r*   key	directoryr)   s         @r   r$   zS3FSWrapper.walk  se      OD1122eeD'::;; 	  	 Cu:D>"k11%%%%^$00		$  1 1 1 1u 1 1 1 2 2 4 4'24 4 4 5 5 K&&&&$ 	= 	=IyyGy<<<<<<<<<<	= 	=r   Nr[   )r^   r_   r`   r   r	   rC   rE   r$   rZ   r   r   r   r   }  sx        S		 	 	 	S	  = = = = = =r   r   c                 \    |                      d          r|                     dd          S | S )Nzs3:// )
startswithreplace)r   s    r   r   r     s/    w ||GR(((r   c                     t          |           }t          |t                    sYdt          j        v r)t          j        d         }t          | |j                  r| S t          d                    |                    | S )NfsspeczUnrecognized filesystem: {})	type
issubclassr	   sysmodules
isinstanceAbstractFileSystemr   format)r   fs_typer   s      r   _ensure_filesystemr     sx    2hhG gz** s{""[*F"f788  	3::7CCDDD	r   c                 r   t          |           s|t          d          || fS |^t          |          }t          |t                    rt          |           }n&t          | t                    st          d          | }||fS t          |           }t          j	        
                    |          }|j        dk    s|j        dk    r|j                            d          }|d         }|dk    rd	}n|j        d
z   |z   }d}t          |          dk    r/|d                                         rt!          |d                   }t"          j                            ||          }|j        }nG|j        dk    r!t                                          }|j        }nt                                          }|}||fS )zn
    Return filesystem from path which could be an HDFS URI, a local URI,
    or a plain filesystem path.
    NzVfilesystem passed but where is file-like, so there is nothing to open with filesystem.zPExpected string path; path-like objects are only allowed with a local filesystemhdfsviewfs:r   r   defaultz://rg   r   )hostportr!   )r   
ValueErrorr   r   rd   r   str	TypeErrorurllibparseurlparseschemenetlocr   r   	isnumericintpar   _connectr   rx   )	whererM   r   
parsed_urinetloc_splitr   r   r   fs_paths	            r   resolve_filesystem_and_pathr     s   
  !! J K K K5  '
33
j/22 	"5))DDE3'' 	*  
 D45!!D&&t,,JF""j&78&C&C!(..s33A2::DD$u,t3D|!!l1o&?&?&A&A!|A''DW4d33/		f	$	$**,,/ **,,w;r   r/   )r   r   r   urllib.parser   rj   os.pathr   r}   pyarrowr   pyarrow.utilr   r   r   r   r   rl   r	   rd   r   r   r   r   r   rZ   r   r   <module>r      s  & 
			     



      ! ! ! ! ! !     G G G G G G G G G G G G  y '+? 
K K K K K K K K\F F F F Fj F F FR?" ?" ?" ?" ?"Z ?" ?" ?"D4= 4= 4= 4= 4=. 4= 4= 4=n    &1 1 1 1 1 1r   