
    'e;                        d Z ddlmZmZ ddlmZmZmZmZm	Z	m
Z
mZmZmZmZmZ eZg Z	 ddlmZ n # e$ r e                    d           Y nw xY w	 ddlmZ n # e$ r e                    d           Y nw xY w	 ddlmZmZmZmZmZmZmZm Z m!Z!m"Z" dd	l#Z# e#j$        e            n # e$ r e                    d
           Y nw xY wd Z%d Z&	 ddZ'	 ddZ(	 dddddZ) G d de          Z*d	S )zO
FileSystem abstraction to interact with various local and remote filesystems.
    )_is_path_like_stringify_path)FileSelectorFileTypeFileInfo
FileSystemLocalFileSystemSubTreeFileSystem_MockFileSystemFileSystemHandlerPyFileSystem_copy_files_copy_files_selector)HadoopFileSystemr   )GcsFileSystemr   )
AwsDefaultS3RetryStrategyAwsStandardS3RetryStrategyS3FileSystem
S3LogLevelS3RetryStrategyensure_s3_initializedfinalize_s3ensure_s3_finalizedinitialize_s3resolve_s3_regionNr   c                     | t           v r"t          d                    |                     t          d                    |                     )Nz<The pyarrow installation is not built with support for '{0}'z*module 'pyarrow.fs' has no attribute '{0}')_not_importedImportErrorformatAttributeError)names    *  /croot/pyarrow_1707330824290/_h_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold/lib/python3.11/site-packages/pyarrow/fs.py__getattr__r#   E   sT    }F4LL
 
 	

 4;;DAA      c                 J   t          j        |           \  }}|                    |          }|rt|                    |g          d         }|j        t
          j        k    r3t          d                    |j        j	        |j
        |                     t          ||          }|S )Nr   zThe path component of the filesystem URI must point to a directory but it has a type: `{}`. The path component is `{}` and the given filesystem URI is `{}`)r   from_urinormalize_pathget_file_infotyper   	Directory
ValueErrorr   r!   pathr
   )uri
filesystemprefixprefix_infos       r"   _filesystem_from_strr1   Q   s     $,S11J&&v..F ; ..x88;x111??Ev$);+;S@ @   'vz::
r$   Fc                 ^   t          | t                    r| S t          | t                    r |rt          d          t	          |           S 	 dd l}t          | |j                  rDt          |           j        dk    rt          |          S t          t          |                     S n# t          $ r Y nw xY wdd lm} t          | |j	                  rt          |          S |rt          | |j                  r| S t          d                    t          |                               )NzXSpecifying to use memory mapping not supported for filesystem specified as an URI stringr   r	   use_mmapzlUnrecognized filesystem: {}. `filesystem` argument must be a FileSystem instance or a valid file system URI')
isinstancer   strr+   r1   fsspecAbstractFileSystemr)   __name__r	   r   FSSpecHandlerr   pyarrow.filesystemr.   	TypeErrorr   )r.   r4   allow_legacy_filesystemr7   legacyfss        r"   _ensure_filesystemr?   e   st    *j)) 0	J	$	$ 0 	8   $J///	; j&";<< 	;J(,===&9999j 9 9:::		;     *)))))*h677 21111 :j(:M#N#N 
	::@&; ;  s   B, ,
B98B9c                    t          |           s|t          d          || fS |vt          |||          }t          |t                    rt          |           } n$t          | t                    st          d          |s|                    |           } || fS t          |           } t	          |          }	 |	                    |           }|j
        t          j        k    }n# t          $ r d}d}Y nw xY w|sS	 t          j        |           \  }} nO# t          $ r-}dt          |          vrdt          |          vr Y d}~nd}~ww xY w|                    |           } || fS )	z`
    Return filesystem/path from path which could be an URI or a plain
    filesystem path.
    Nzg'filesystem' passed but the specified path is file-like, so there is nothing to open with 'filesystem'.)r4   r=   zPExpected string path; path-like objects are only allowed with a local filesystemr3   Fzempty schemezCannot parse URI)r   r+   r?   r5   r	   r   r6   r<   r'   r(   r)   r   NotFoundr   r&   )r,   r.   r=   
memory_map	file_infoexists_locallyes          r"   _resolve_filesystem_and_pathrF      s      !?   4'$;
 
 

 j/22 	"4((DDD#&& 	*   ' 	3,,T22D44  D
 !*555J?,,T22	
 $.H,==	    	  /	)2488J 	 	 	 SVV++*#a&&88	 ((..ts*   =C( (C98C9?D 
E!#E		Ei   T)
chunk_sizeuse_threadsc                   t          | |          \  }}t          ||          \  }}	|                    |          }
|
j        t          j        k    r't          |d          }t          ||||	||           dS t          ||||	||           dS )a  
    Copy files between FileSystems.

    This functions allows you to recursively copy directories of files from
    one file system to another, such as from S3 to your local machine.

    Parameters
    ----------
    source : string
        Source file path or URI to a single file or directory.
        If a directory, files will be copied recursively from this path.
    destination : string
        Destination file path or URI. If `source` is a file, `destination`
        is also interpreted as the destination file (not directory).
        Directories will be created as necessary.
    source_filesystem : FileSystem, optional
        Source filesystem, needs to be specified if `source` is not a URI,
        otherwise inferred.
    destination_filesystem : FileSystem, optional
        Destination filesystem, needs to be specified if `destination` is not
        a URI, otherwise inferred.
    chunk_size : int, default 1MB
        The maximum size of block to read before flushing to the
        destination file. A larger chunk_size will use more memory while
        copying but may help accommodate high latency FileSystems.
    use_threads : bool, default True
        Whether to use multiple threads to accelerate copying.

    Examples
    --------
    Inspect an S3 bucket's files:

    >>> s3, path = fs.FileSystem.from_uri(
    ...            "s3://registry.opendata.aws/roda/ndjson/")
    >>> selector = fs.FileSelector(path)
    >>> s3.get_file_info(selector)
    [<FileInfo for 'registry.opendata.aws/roda/ndjson/index.ndjson':...]

    Copy one file from S3 bucket to a local directory:

    >>> fs.copy_files("s3://registry.opendata.aws/roda/ndjson/index.ndjson",
    ...               "file:///{}/index_copy.ndjson".format(local_path))

    >>> fs.LocalFileSystem().get_file_info(str(local_path)+
    ...                                    '/index_copy.ndjson')
    <FileInfo for '.../index_copy.ndjson': type=FileType.File, size=...>

    Copy file using a FileSystem object:

    >>> fs.copy_files("registry.opendata.aws/roda/ndjson/index.ndjson",
    ...               "file:///{}/index_copy.ndjson".format(local_path),
    ...               source_filesystem=fs.S3FileSystem())
    T	recursiveN)rF   r(   r)   r   r*   r   r   r   )sourcedestinationsource_filesystemdestination_filesystemrG   rH   	source_fssource_pathdestination_fsdestination_pathrC   
source_sels               r"   
copy_filesrU      s    p :! I{ (D+( ($N$ ''44I~+++!+>>>
Y
+-='	6 	6 	6 	6 	6 	I{"$4	- 	- 	- 	- 	-r$   c                       e Zd ZdZd Zd Zd Zd Zd Ze	d             Z
d Zd	 Zd
 Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd ZdS )r:   a  
    Handler for fsspec-based Python filesystems.

    https://filesystem-spec.readthedocs.io/en/latest/index.html

    Parameters
    ----------
    fs : FSSpec-compliant filesystem instance

    Examples
    --------
    >>> PyFileSystem(FSSpecHandler(fsspec_fs)) # doctest: +SKIP
    c                     || _         d S N)fs)selfrY   s     r"   __init__zFSSpecHandler.__init__(  s    r$   c                 Z    t          |t                    r| j        |j        k    S t          S rX   r5   r:   rY   NotImplementedrZ   others     r"   __eq__zFSSpecHandler.__eq__+  )    e]++ 	'7eh&&r$   c                 Z    t          |t                    r| j        |j        k    S t          S rX   r]   r_   s     r"   __ne__zFSSpecHandler.__ne__0  rb   r$   c                 ~    | j         j        }t          |t                    r|d         }d                    |          S )Nr   z
fsspec+{0})rY   protocolr5   listr   )rZ   rf   s     r"   get_type_namezFSSpecHandler.get_type_name5  s;    7#h%% 	#{H""8,,,r$   c                     |S rX    rZ   r,   s     r"   r'   zFSSpecHandler.normalize_path;  s    r$   c           	          |d         }|d         dk    rt           j        }n'|d         dk    rt           j        }d }nt           j        }t	          | |||                    dd                     S )Nsizer)   file	directorymtime)rm   rp   )r   Filer*   Unknownr   get)r,   inform   ftypes       r"   _create_file_infozFSSpecHandler._create_file_info>  sn    F|<6!!MEE&\[((&EDD$Ee$dhhw6M6MNNNNr$   c                    g }|D ]}	 | j                             |          }|                    |                     ||                     G# t          $ r0 |                    t          |t          j                             Y w xY w|S rX   )rY   rt   appendrv   FileNotFoundErrorr   r   rA   )rZ   pathsinfosr,   rt   s        r"   r(   zFSSpecHandler.get_file_infoK  s     	A 	ADAw||D)) T33D$??@@@@ % @ @ @XdH,=>>?????@ s   A7BBc                 6   | j                             |j                  sP| j                             |j                  rt	          |j                  |j        rg S t          |j                  |j        rd }nd}g }| j                             |j        |dd          }|	                                D ]c\  }}|
                    d          }|j        
                    d          }||k    r)|                    |                     ||                     d|S )N   T)maxdepthwithdirsdetail/)rY   isdirbase_direxistsNotADirectoryErrorallow_not_foundry   rK   finditemsstriprx   rv   )	rZ   selectorr~   r{   selected_filesr,   rt   _pathr   s	            r"   get_file_info_selectorz$FSSpecHandler.get_file_info_selectorV  s%   w}}X.// 	?w~~h/00 ?():;;;+ ?I+H,=>>> 	HHH4 & 
 
 )..00 	A 	AJD$JJsOOE(..s33H   T33D$??@@@r$   c                 b    	 | j                             ||           d S # t          $ r Y d S w xY w)N)create_parents)rY   mkdirFileExistsError)rZ   r,   rK   s      r"   
create_dirzFSSpecHandler.create_dirs  sG    	GMM$yM99999 	 	 	DD	s     
..c                 >    | j                             |d           d S NTrJ   )rY   rmrk   s     r"   
delete_dirzFSSpecHandler.delete_dirz  s     

44
(((((r$   c                 J   	 | j                             |d          }n# t          $ r |rY d S  w xY w|D ]m}| j                             |          r| j                             |d           9| j                             |          r| j                             |           nd S )NF)r   TrJ   )rY   listdirry   r   r   isfile)rZ   r,   missing_dir_oksubpathssubpaths        r"   _delete_dir_contentsz"FSSpecHandler._delete_dir_contents}  s    	wtE::HH  	 	 	 	   	$ 	$Gw}}W%% $

7d
3333(( $

7###		$ 	$s    00c                     |                     d          dk    rt          d|d          |                     ||           d S )Nr    z$delete_dir_contents called on path '')r   r+   r   )rZ   r,   r   s      r"   delete_dir_contentsz!FSSpecHandler.delete_dir_contents  sO    ::c??b  6cC C C!!$77777r$   c                 0    |                      d           d S )Nr   )r   )rZ   s    r"   delete_root_dir_contentsz&FSSpecHandler.delete_root_dir_contents  s    !!#&&&&&r$   c                     | j                             |          st          |          | j                             |           d S rX   )rY   r   ry   r   rk   s     r"   delete_filezFSSpecHandler.delete_file  sB     w~~d## 	*#D)))

4r$   c                 @    | j                             ||d           d S r   )rY   mvrZ   srcdests      r"   movezFSSpecHandler.move  s"    

3
-----r$   c                 <    | j                             ||           d S rX   )rY   copyr   s      r"   	copy_filezFSSpecHandler.copy_file  s      	S$r$   c                     ddl m} | j                            |          st	          |           || j                            |d          d          S Nr   
PythonFilerbmoderpyarrowr   rY   r   ry   openrZ   r,   r   s      r"   open_input_streamzFSSpecHandler.open_input_stream  _    &&&&&&w~~d## 	*#D)))z$',,t$,77cBBBBr$   c                     ddl m} | j                            |          st	          |           || j                            |d          d          S r   r   r   s      r"   open_input_filezFSSpecHandler.open_input_file  r   r$   c                 \    ddl m}  || j                            |d          d          S )Nr   r   wbr   wr   r   rY   r   rZ   r,   metadatar   s       r"   open_output_streamz FSSpecHandler.open_output_stream  ;    &&&&&&z$',,t$,77cBBBBr$   c                 \    ddl m}  || j                            |d          d          S )Nr   r   abr   r   r   r   s       r"   open_append_streamz FSSpecHandler.open_append_stream  r   r$   N)r9   
__module____qualname____doc__r[   ra   rd   rh   r'   staticmethodrv   r(   r   r   r   r   r   r   r   r   r   r   r   r   r   rj   r$   r"   r:   r:     sg            
  
- - -   
O 
O \
O	 	 	  :  ) ) )$ $ $8 8 8' ' '  . . .     C C CC C CC C C
C C C C Cr$   r:   )FF)NFF)NN)+r   pyarrow.utilr   r   pyarrow._fsr   r   r   r   r	   r
   r   r   r   r   r   	FileStatsr   pyarrow._hdfsr   r   rx   pyarrow._gcsfsr   pyarrow._s3fsr   r   r   r   r   r   r   r   r   r   atexitregisterr#   r1   r?   rF   rU   r:   rj   r$   r"   <module>r      s  $  8 7 7 7 7 7 7 7                          	-....... - - -+,,,,,-*,,,,,,, * * *)))))*)L L L L L L L L L L L L L L L L L L L L L L L L MMMFO'((((  ) ) )((((()	 	 	  * 9>& & & &T FK= = = =B ?CH-&DH- H- H- H- H-VcC cC cC cC cC% cC cC cC cC cCs3   1 AAA A65A6:B' 'CC