
    'e8                         d dl Z d dlZd dlZd dlZd dlZd dlmZ d dl	Z
d dlmZmZmZmZ  ej                   ddd      Z G d d      Z G d	 d
e      Z G d de      Z G d de      Zd Zd ZddZy)    N)join)doc_stringify_path_is_path_like	_DEPR_MSGzfilesystem.LocalFileSystemz2.0.0zfs.LocalFileSystemc                       e Zd ZdZd Zd ZddZd Zd Zd Z	ddZ
d	 Zd
 ZddZd Zd Zd Zd Z	 	 ddZddZed        Zy)
FileSystemz(
    Abstract filesystem interface.
    c                 r    | j                  |d      5 }|j                         cddd       S # 1 sw Y   yxY w)z
        Return contents of file as a bytes object.

        Parameters
        ----------
        path : str
            File path to read content from.

        Returns
        -------
        contents : bytes
        rbN)openread)selfpathfs      2lib/python3.11/site-packages/pyarrow/filesystem.pycatzFileSystem.cat)   s2     YYtT" 	a668	 	 	s   -6c                     t         )z
        Return list of file paths.

        Parameters
        ----------
        path : str
            Directory to list contents from.
        NotImplementedErrorr   r   s     r   lszFileSystem.ls9   
     "!    c                     t         )z
        Delete the indicated file or directory.

        Parameters
        ----------
        path : str
            Path to delete.
        recursive : bool, default False
            If True, also delete child paths for directories.
        r   r   r   	recursives      r   deletezFileSystem.deleteD   
     "!r   c                     t        |      }| j                  |      }|d   dk(  r|d   S d}| j                  |      D ]6  \  }}}|D ]+  }| j                  ||      }|| j                  |      d   z  }- 8 |S )z
        Compute bytes used by all contents under indicated path in file tree.

        Parameters
        ----------
        path : str
            Can be a file path or directory.

        Returns
        -------
        usage : int
        kindfilesizer   )r   statwalk
_path_join)	r   r   	path_infototalrootdirectoriesfiles
child_pathabspaths	            r   
disk_usagezFileSystem.disk_usageQ   s     t$IIdO	V&V$$(,		$ 	4$D+u# 4
//$
;7+F334	4
 r   c                 8    | j                   j                  |      S N)pathsepr   )r   argss     r   r%   zFileSystem._path_joink   s    ||  &&r   c                     t        d      )zl
        Information about a filesystem entry.

        Returns
        -------
        stat : dict
        zFileSystem.statr   r   s     r   r#   zFileSystem.statn   s     ""344r   c                 (    | j                  ||      S )z.
        Alias for FileSystem.delete.
        r   )r   r   s      r   rmzFileSystem.rmx   s     {{49{55r   c                 &    | j                  ||      S )z.
        Alias for FileSystem.rename.
        )renamer   r   new_paths      r   mvzFileSystem.mv~   s     {{4**r   c                     t        d      )z
        Rename file, like UNIX mv command.

        Parameters
        ----------
        path : str
            Path to alter.
        new_path : str
            Path to move to.
        zFileSystem.renamer   r8   s      r   r7   zFileSystem.rename   s     ""566r   c                     t         )z
        Create a directory.

        Parameters
        ----------
        path : str
            Path to the directory.
        create_parents : bool, default True
            If the parent directories don't exists create them as well.
        r   r   r   create_parentss      r   mkdirzFileSystem.mkdir   r   r   c                     t         )z
        Return True if path exists.

        Parameters
        ----------
        path : str
            Path to check.
        r   r   s     r   existszFileSystem.exists   r   r   c                     t         )z
        Return True if path is a directory.

        Parameters
        ----------
        path : str
            Path to check.
        r   r   s     r   isdirzFileSystem.isdir   r   r   c                     t         )z
        Return True if path is a file.

        Parameters
        ----------
        path : str
            Path to check.
        r   r   s     r   isfilezFileSystem.isfile   r   r   c                     t         )zf
        Returns True if this FileSystem is a unix-style file store with
        directories.
        r   r   s    r   _isfilestorezFileSystem._isfilestore   s
    
 "!r   Nc                 N    ddl m}  |||||       }|j                  |||      S )a]  
        Read Parquet data from path in file system. Can read from a single file
        or a directory of files.

        Parameters
        ----------
        path : str
            Single file path or directory
        columns : List[str], optional
            Subset of columns to read.
        metadata : pyarrow.parquet.FileMetaData
            Known metadata to validate files against.
        schema : pyarrow.parquet.Schema
            Known schema to validate files against. Alternative to metadata
            argument.
        use_threads : bool, default True
            Perform multi-threaded column reads.
        use_pandas_metadata : bool, default False
            If True and file has custom pandas schema metadata, ensure that
            index columns are also loaded.

        Returns
        -------
        table : pyarrow.Table
        r   )ParquetDataset)schemametadata
filesystem)columnsuse_threadsuse_pandas_metadata)pyarrow.parquetrJ   r   )	r   r   rN   rL   rK   rO   rP   rJ   datasets	            r   read_parquetzFileSystem.read_parquet   s9    6 	3 fx,02||G0C  E 	Er   c                     t         )3
        Open file for reading or writing.
        r   r   r   modes      r   r   zFileSystem.open   s
     "!r   c                      y)N/ rG   s    r   r0   zFileSystem.pathsep   s    r   FT)NNNTFr   )__name__
__module____qualname____doc__r   r   r   r-   r%   r#   r5   r:   r7   r?   rA   rC   rE   rH   rS   r   propertyr0   rZ   r   r   r	   r	   $   sw     	""4'56+7"	"	"	"" FJ;@EB"  r   r	   c                       e Zd ZdZ fdZed        Zed        Z ee	j                        d        Z
 ee	j                        dd       Z ee	j                        d        Z ee	j                        d        Z ee	j                        d	        Z ee	j                        d
        Z ee	j                         dd       Zed        Zd Z xZS )LocalFileSystemNc                 b    t        j                  t        t        d       t        |           y N   
stacklevel)warningswarn_FS_DEPR_MSGFutureWarningsuper__init__)r   	__class__s    r   ro   zLocalFileSystem.__init__   s    lMa@r   c                     | j                   Mt        j                         5  t        j                  d       t	               | _         d d d        | j                   S | j                   S # 1 sw Y   | j                   S xY w)Nignore)	_instancerj   catch_warningssimplefilterrd   clss    r   _get_instancezLocalFileSystem._get_instance   sa    == ((* 2%%h/ / 12 }}s}}2 }}s   %A&&A:c                 b    t        j                  t        t        d       | j	                         S rf   )rj   rk   rl   rm   rx   rv   s    r   get_instancezLocalFileSystem.get_instance  s!    lMa@  ""r   c                 h    t              t        fdt        j                        D              S )Nc              3   6   K   | ]  }t        |        y wr/   )pjoin).0xr   s     r   	<genexpr>z%LocalFileSystem.ls.<locals>.<genexpr>
  s     ?eD!n?s   )r   sortedoslistdirr   s    `r   r   zLocalFileSystem.ls  s'    t$?bjj.>???r   c                 t    t        |      }|rt        j                  |       y t        j                  |       y r/   )r   r   makedirsr?   r=   s      r   r?   zLocalFileSystem.mkdir  s&    t$KKHHTNr   c                 V    t        |      }t        j                  j                  |      S r/   )r   r   r   rC   r   s     r   rC   zLocalFileSystem.isdir  s    t$ww}}T""r   c                 V    t        |      }t        j                  j                  |      S r/   )r   r   r   rE   r   s     r   rE   zLocalFileSystem.isfile      t$ww~~d##r   c                      y)NTrZ   rG   s    r   rH   zLocalFileSystem._isfilestore  s    r   c                 V    t        |      }t        j                  j                  |      S r/   )r   r   r   rA   r   s     r   rA   zLocalFileSystem.exists"  r   r   c                 2    t        |      }t        ||      S rU   )rW   )r   r   rV   s      r   r   zLocalFileSystem.open'  s    
 t$Dt$$r   c                 6    t         j                  j                  S r/   )r   r   seprG   s    r   r0   zLocalFileSystem.pathsep/  s    ww{{r   c                 B    t        |      }t        j                  |      S )z8
        Directory tree generator, see os.walk.
        )r   r   r$   r   s     r   r$   zLocalFileSystem.walk3  s     t$wwt}r   r\   r]   )r^   r_   r`   rs   ro   classmethodrx   rz   r   r	   r   r?   rC   rE   rH   rA   r   rb   r0   r$   __classcell__)rp   s   @r   rd   rd      s%   I   # # 	@ @ 			  			# # 			$ $ 		 	 ! " 			$ $ 	% %  r   rd   c                   z   e Zd ZdZd Z eej                        d        Z eej                        d        Z eej                        d        Z	 eej                        dd       Z
 eej                        d        Z eej                        dd       Z eej                        dd	       Zdd
Zd Zy)DaskFileSystemzJ
    Wraps s3fs Dask filesystem implementation like s3fs, gcsfs, etc.
    c                 J    t        j                  dt        d       || _        y )Nz~The pyarrow.filesystem.DaskFileSystem/S3FSWrapper are deprecated as of pyarrow 3.0.0, and will be removed in a future version.rg   rh   )rj   rk   rm   fs)r   r   s     r   ro   zDaskFileSystem.__init__@  s"    La	) r   c                     t        d      NzUnsupported file system APIr   r   s     r   rC   zDaskFileSystem.isdirG      !"?@@r   c                     t        d      r   r   r   s     r   rE   zDaskFileSystem.isfileK  r   r   c                      y)zh
        Object Stores like S3 and GCSFS are based on key lookups, not true
        file-paths.
        FrZ   rG   s    r   rH   zDaskFileSystem._isfilestoreO  s     r   c                 R    t        |      }| j                  j                  ||      S )Nr4   )r   r   r5   r   s      r   r   zDaskFileSystem.deleteW  s#    t$wwzz$)z44r   c                 N    t        |      }| j                  j                  |      S r/   )r   r   rA   r   s     r   rA   zDaskFileSystem.exists\  s    t$ww~~d##r   c                     t        |      }|r| j                  j                  |      S | j                  j                  |      S r/   )r   r   mkdirsr?   r=   s      r   r?   zDaskFileSystem.mkdira  s5    t$77>>$''77==&&r   c                 R    t        |      }| j                  j                  ||      S r   )r   r   r   rV   s      r   r   zDaskFileSystem.openi  s%    
 t$ww||Dt|,,r   c                 R    t        |      }| j                  j                  ||      S )N)detail)r   r   r   )r   r   r   s      r   r   zDaskFileSystem.lsq  s#    t$wwzz$vz..r   c                 N    t        |      }| j                  j                  |      S )z9
        Directory tree generator, like os.walk.
        )r   r   r$   r   s     r   r$   zDaskFileSystem.walku  s!     t$ww||D!!r   Nr[   r\   r]   )r^   r_   r`   ra   ro   r   r	   rC   rE   rH   r   rA   r?   r   r   r$   rZ   r   r   r   r   ;  s     			A A 			A A 		 	 ! " 			5 5 			$ $ 			' ' 	- -/"r   r   c                   t    e Zd Z eej
                        d        Z eej                        d        ZddZy)S3FSWrapperc                     t        t        |            }	 | j                  j                  |      }t	        |      dk(  r	|d   |k(  ryy# t
        $ r Y yw xY w)N   r   FT_sanitize_s3r   r   r   lenOSErrorr   r   contentss      r   rC   zS3FSWrapper.isdir  sW    OD12	wwzz$'H8}!hqkT&9 		s   1A	 		AAc                     t        t        |            }	 | j                  j                  |      }t	        |      dk(  xr |d   |k(  S # t
        $ r Y yw xY w)Nr   r   Fr   r   s      r   rE   zS3FSWrapper.isfile  sW    OD12	wwzz$'Hx=A%=(1+*== 		s   2A	 		AAc              #   R  K   t        t        |            }t               }t               }t        | j                  j                  ||            D ];  }|d   }|d   dk(  r|j                  |       "|d   dk(  r+|j                  |       = t        |D cg c]  }||vrt        j                  |      d     c}      }t        |D cg c]  }t        j                  |      d    c}      }|||f |D ]  }| j                  ||      E d{     yc c}w c c}w 7 w)z
        Directory tree generator, like os.walk.

        Generator version of what is in s3fs, which yields a flattened list of
        files.
        )refreshKeyStorageClass	DIRECTORYBUCKETr   N)r   r   setlistr   _lsaddr   	posixpathsplitr$   )	r   r   r   r)   r*   keyr   r   	directorys	            r   r$   zS3FSWrapper.walk  s.     OD12eD':; 	 Cu:D>"k1%^$0		$	  u 1!K/ "*1- 1 2'24"# (ooa03 4 5 K&&$ 	=IyyGy<<<	=14 =s*   BD'#D;D'D *)D'D%D'Nr[   )r^   r_   r`   r   r	   rC   rE   r$   rZ   r   r   r   r   }  sC    			 	 			 =r   r   c                 L    | j                  d      r| j                  dd      S | S )Nzs3:// )
startswithreplace)r   s    r   r   r     s$    w||GR((r   c                     t        |       }t        |t              sWdt        j                  v r+t        j                  d   }t        | |j                        r| S t        dj                  |            | S )NfsspeczUnrecognized filesystem: {})	type
issubclassr	   sysmodules
isinstanceAbstractFileSystemr   format)r   fs_typer   s      r   _ensure_filesystemr     sd    2hG gz*s{{"[[*F"f778 	3::7CDD	r   c                 6   t        |       s|t        d      || fS |Kt        |      }t        |t              rt        |       }||fS t        | t              st        d      | }||fS t        |       }t        j                  j                  |      }|j                  dk(  s|j                  dk(  r|j                  j                  d      }|d   }|dk(  rd}n|j                  d	z   |z   }d}t        |      d
k(  r!|d   j                         rt!        |d         }t"        j$                  j'                  ||      }|j(                  }||fS |j                  dk(  r$t        j+                         }|j(                  }||fS t        j+                         }|}||fS )zn
    Return filesystem from path which could be an HDFS URI, a local URI,
    or a plain filesystem path.
    zVfilesystem passed but where is file-like, so there is nothing to open with filesystem.zPExpected string path; path-like objects are only allowed with a local filesystemhdfsviewfs:r   r   defaultz://rg   r   )hostportr!   )r   
ValueErrorr   r   rd   r   str	TypeErrorurllibparseurlparseschemenetlocr   r   	isnumericintpar   _connectr   rx   )	whererM   r   
parsed_urinetloc_splitr   r   r   fs_paths	            r   resolve_filesystem_and_pathr     s   
 ! J K K5  '
3
j/2"5)D 4 E3'* 
 D45!D&&t,JF"j&7&78&C!((..s3A2:D$$u,t3D|!l1o&?&?&A|A'DWW4d3// w; 
		f	$**,// w; **,w;r   r/   )r   r   r   urllib.parser   rj   os.pathr   r}   pyarrowr   pyarrow.utilr   r   r   r   r   rl   r	   rd   r   r   r   r   r   rZ   r   r   <module>r      s   & 
  
   !  G G  y '+?
K K\Fj FR?"Z ?"D4=. 4=n&1r   