
    'e;                        d Z ddlmZmZ ddlmZmZmZmZm	Z	m
Z
mZmZmZmZmZ eZg Z	 ddlmZ 	 ddlmZ 	 ddlmZmZmZmZmZmZmZm Z m!Z!m"Z" dd	l#Z# e#jH                  e        d Z%d Z&	 ddZ'	 ddZ(	 dddddZ) G d de      Z*y	# e$ r ej+                  d       Y sw xY w# e$ r ej+                  d       Y w xY w# e$ r ej+                  d
       Y uw xY w)zO
FileSystem abstraction to interact with various local and remote filesystems.
    )_is_path_like_stringify_path)FileSelectorFileTypeFileInfo
FileSystemLocalFileSystemSubTreeFileSystem_MockFileSystemFileSystemHandlerPyFileSystem_copy_files_copy_files_selector)HadoopFileSystemr   )GcsFileSystemr   )
AwsDefaultS3RetryStrategyAwsStandardS3RetryStrategyS3FileSystem
S3LogLevelS3RetryStrategyensure_s3_initializedfinalize_s3ensure_s3_finalizedinitialize_s3resolve_s3_regionNr   c                 z    | t         v rt        dj                  |             t        dj                  |             )Nz<The pyarrow installation is not built with support for '{0}'z*module 'pyarrow.fs' has no attribute '{0}')_not_importedImportErrorformatAttributeError)names    *lib/python3.11/site-packages/pyarrow/fs.py__getattr__r#   E   sB    }F4L
 	

 4;;DA     c                 L   t        j                  |       \  }}|j                  |      }|rx|j                  |g      d   }|j                  t
        j                  k7  r:t        dj                  |j                  j                  |j                  |             t        ||      }|S )Nr   zThe path component of the filesystem URI must point to a directory but it has a type: `{}`. The path component is `{}` and the given filesystem URI is `{}`)r   from_urinormalize_pathget_file_infotyper   	Directory
ValueErrorr   r!   pathr
   )uri
filesystemprefixprefix_infos       r"   _filesystem_from_strr1   Q   s     $,,S1J&&v.F ..x8;x111??Ev$$));+;+;S@  'vz:
r$   c                    t        | t              r| S t        | t              r|rt        d      t	        |       S 	 dd l}t        | |j                        r8t        |       j                  dk(  rt        |      S t        t        |             S dd lm} t        | |j                        rt        |      S |rt        | |j                        r| S t        dj!                  t        |                   # t        $ r Y pw xY w)NzXSpecifying to use memory mapping not supported for filesystem specified as an URI stringr   r	   use_mmapzlUnrecognized filesystem: {}. `filesystem` argument must be a FileSystem instance or a valid file system URI')
isinstancer   strr+   r1   fsspecAbstractFileSystemr)   __name__r	   r   FSSpecHandlerr   pyarrow.filesystemr.   	TypeErrorr   )r.   r4   allow_legacy_filesystemr7   legacyfss        r"   _ensure_filesystemr?   e   s     *j)	J	$8  $J//	; j&";";<J((,==&99j 9:: **h66711:j(:M:M#N
	::@&; %  s   C3 3	C?>C?c                 x   t        |       s|t        d      || fS |\t        |||      }t        |t              rt        |       } nt        | t              st        d      |s|j                  |       } || fS t        |       } t	        |      }	 |j                  |       }|j                  t        j                  k7  }|s	 t        j                  |       \  }} || fS |j                  |       } || fS # t        $ r d}d}Y Cw xY w# t        $ r(}dt        |      vrdt        |      vr Y d}~|| fS d}~ww xY w)	z`
    Return filesystem/path from path which could be an URI or a plain
    filesystem path.
    Nzg'filesystem' passed but the specified path is file-like, so there is nothing to open with 'filesystem'.)r4   r=   zPExpected string path; path-like objects are only allowed with a local filesystemr3   Fzempty schemezCannot parse URI)r   r+   r?   r5   r	   r   r6   r<   r'   r(   r)   r   NotFoundr   r&   )r,   r.   r=   
memory_map	file_infoexists_locallyes          r"   _resolve_filesystem_and_pathrF      s|    !?  4'$;

 j/2"4(DD#&*  ',,T2D44 D
 !*5J?,,T2	
 $..H,=,== 	)2248J t ((.t+  	  	 SV+*#a&8 t	s*   C5 D 5DD	D9D44D9i   T)
chunk_sizeuse_threadsc                    t        | |      \  }}t        ||      \  }}	|j                  |      }
|
j                  t        j                  k(  rt        |d      }t        ||||	||       yt        ||||	||       y)a  
    Copy files between FileSystems.

    This functions allows you to recursively copy directories of files from
    one file system to another, such as from S3 to your local machine.

    Parameters
    ----------
    source : string
        Source file path or URI to a single file or directory.
        If a directory, files will be copied recursively from this path.
    destination : string
        Destination file path or URI. If `source` is a file, `destination`
        is also interpreted as the destination file (not directory).
        Directories will be created as necessary.
    source_filesystem : FileSystem, optional
        Source filesystem, needs to be specified if `source` is not a URI,
        otherwise inferred.
    destination_filesystem : FileSystem, optional
        Destination filesystem, needs to be specified if `destination` is not
        a URI, otherwise inferred.
    chunk_size : int, default 1MB
        The maximum size of block to read before flushing to the
        destination file. A larger chunk_size will use more memory while
        copying but may help accommodate high latency FileSystems.
    use_threads : bool, default True
        Whether to use multiple threads to accelerate copying.

    Examples
    --------
    Inspect an S3 bucket's files:

    >>> s3, path = fs.FileSystem.from_uri(
    ...            "s3://registry.opendata.aws/roda/ndjson/")
    >>> selector = fs.FileSelector(path)
    >>> s3.get_file_info(selector)
    [<FileInfo for 'registry.opendata.aws/roda/ndjson/index.ndjson':...]

    Copy one file from S3 bucket to a local directory:

    >>> fs.copy_files("s3://registry.opendata.aws/roda/ndjson/index.ndjson",
    ...               "file:///{}/index_copy.ndjson".format(local_path))

    >>> fs.LocalFileSystem().get_file_info(str(local_path)+
    ...                                    '/index_copy.ndjson')
    <FileInfo for '.../index_copy.ndjson': type=FileType.File, size=...>

    Copy file using a FileSystem object:

    >>> fs.copy_files("registry.opendata.aws/roda/ndjson/index.ndjson",
    ...               "file:///{}/index_copy.ndjson".format(local_path),
    ...               source_filesystem=fs.S3FileSystem())
    T	recursiveN)rF   r(   r)   r   r*   r   r   r   )sourcedestinationsource_filesystemdestination_filesystemrG   rH   	source_fssource_pathdestination_fsdestination_pathrC   
source_sels               r"   
copy_filesrU      s    p :!I{ (D+($N$ ''4I~~+++!+>
Y
+-='	6 	I{"$4	-r$   c                       e Zd ZdZd Zd Zd Zd Zd Ze	d        Z
d Zd	 Zd
 Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zy)r:   a  
    Handler for fsspec-based Python filesystems.

    https://filesystem-spec.readthedocs.io/en/latest/index.html

    Parameters
    ----------
    fs : FSSpec-compliant filesystem instance

    Examples
    --------
    >>> PyFileSystem(FSSpecHandler(fsspec_fs)) # doctest: +SKIP
    c                     || _         y N)fs)selfrY   s     r"   __init__zFSSpecHandler.__init__(  s	    r$   c                 `    t        |t              r| j                  |j                  k(  S t        S rX   r5   r:   rY   NotImplementedrZ   others     r"   __eq__zFSSpecHandler.__eq__+  %    e]+77ehh&&r$   c                 `    t        |t              r| j                  |j                  k7  S t        S rX   r]   r_   s     r"   __ne__zFSSpecHandler.__ne__0  rb   r$   c                 z    | j                   j                  }t        |t              r|d   }dj	                  |      S )Nr   z
fsspec+{0})rY   protocolr5   listr   )rZ   rf   s     r"   get_type_namezFSSpecHandler.get_type_name5  s5    77##h%{H""8,,r$   c                     |S rX    rZ   r,   s     r"   r'   zFSSpecHandler.normalize_path;  s    r$   c           	          |d   }|d   dk(  rt         j                  }n+|d   dk(  rt         j                  }d }nt         j                  }t	        | |||j                  dd             S )Nsizer)   file	directorymtime)rm   rp   )r   Filer*   Unknownr   get)r,   inform   ftypes       r"   _create_file_infozFSSpecHandler._create_file_info>  sc    F|<6!MME&\[(&&ED$$Ee$dhhw6MNNr$   c                    g }|D ]?  }	 | j                   j                  |      }|j                  | j                  ||             A |S # t        $ r, |j                  t        |t        j                               Y xw xY wrX   )rY   rt   appendrv   FileNotFoundErrorr   r   rA   )rZ   pathsinfosr,   rt   s        r"   r(   zFSSpecHandler.get_file_infoK  s~     	ADAww||D) T33D$?@	A 	 % @XdH,=,=>?@s   A		2A>=A>c                 X   | j                   j                  |j                        s]| j                   j                  |j                        rt	        |j                        |j
                  rg S t        |j                        |j                  rd }nd}g }| j                   j                  |j                  |dd      }|j                         D ]X  \  }}|j                  d      }|j                  j                  d      }||k7  s8|j                  | j                  ||             Z |S )N   T)maxdepthwithdirsdetail/)rY   isdirbase_direxistsNotADirectoryErrorallow_not_foundry   rK   finditemsstriprx   rv   )	rZ   selectorr~   r{   selected_filesr,   rt   _pathr   s	            r"   get_file_info_selectorz$FSSpecHandler.get_file_info_selectorV  s	   ww}}X../ww~~h//0():):;;++I+H,=,=>>HH4 & 
 )..0 	AJD$JJsOE((..s3H  T33D$?@	A r$   c                 ^    	 | j                   j                  ||       y # t        $ r Y y w xY w)N)create_parents)rY   mkdirFileExistsError)rZ   r,   rK   s      r"   
create_dirzFSSpecHandler.create_dirs  s-    	GGMM$yM9 		s     	,,c                 >    | j                   j                  |d       y NTrJ   )rY   rmrk   s     r"   
delete_dirzFSSpecHandler.delete_dirz  s    

44
(r$   c                 R   	 | j                   j                  |d      }|D ]r  }| j                   j                  |      r| j                   j	                  |d       <| j                   j                  |      sX| j                   j	                  |       t y # t        $ r |rY y  w xY w)NF)r   TrJ   )rY   listdirry   r   r   isfile)rZ   r,   missing_dir_oksubpathssubpaths        r"   _delete_dir_contentsz"FSSpecHandler._delete_dir_contents}  s    	wwtE:H
   	$Gww}}W%

7d
3(

7#		$	 ! 		s   B B&$B&c                 j    |j                  d      dk(  rt        d|d      | j                  ||       y )Nr    z$delete_dir_contents called on path '')r   r+   r   )rZ   r,   r   s      r"   delete_dir_contentsz!FSSpecHandler.delete_dir_contents  s8    ::c?b 6cC C!!$7r$   c                 &    | j                  d       y )Nr   )r   )rZ   s    r"   delete_root_dir_contentsz&FSSpecHandler.delete_root_dir_contents  s    !!#&r$   c                     | j                   j                  |      st        |      | j                   j                  |       y rX   )rY   r   ry   r   rk   s     r"   delete_filezFSSpecHandler.delete_file  s/     ww~~d##D))

4r$   c                 @    | j                   j                  ||d       y r   )rY   mvrZ   srcdests      r"   movezFSSpecHandler.move  s    

3
-r$   c                 <    | j                   j                  ||       y rX   )rY   copyr   s      r"   	copy_filezFSSpecHandler.copy_file  s     	S$r$   c                     ddl m} | j                  j                  |      st	        |       || j                  j                  |d      d      S Nr   
PythonFilerbmoderpyarrowr   rY   r   ry   openrZ   r,   r   s      r"   open_input_streamzFSSpecHandler.open_input_stream  >    &ww~~d##D))$'',,t$,7cBBr$   c                     ddl m} | j                  j                  |      st	        |       || j                  j                  |d      d      S r   r   r   s      r"   open_input_filezFSSpecHandler.open_input_file  r   r$   c                 X    ddl m}  || j                  j                  |d      d      S )Nr   r   wbr   wr   r   rY   r   rZ   r,   metadatar   s       r"   open_output_streamz FSSpecHandler.open_output_stream  #    &$'',,t$,7cBBr$   c                 X    ddl m}  || j                  j                  |d      d      S )Nr   r   abr   r   r   r   s       r"   open_append_streamz FSSpecHandler.open_append_stream  r   r$   N)r9   
__module____qualname____doc__r[   ra   rd   rh   r'   staticmethodrv   r(   r   r   r   r   r   r   r   r   r   r   r   r   r   rj   r$   r"   r:   r:     s    

- 
O 
O	:)$8'. CCC
Cr$   r:   )FF)NFF)NN)+r   pyarrow.utilr   r   pyarrow._fsr   r   r   r   r	   r
   r   r   r   r   r   	FileStatsr   pyarrow._hdfsr   r   rx   pyarrow._gcsfsr   pyarrow._s3fsr   r   r   r   r   r   r   r   r   r   atexitregisterr#   r1   r?   rF   rU   r:   rj   r$   r"   <module>r      s  $ 8    	-.*,)L L L FOO'(	* 9>&T FK=B ?CH-&DH-VcC% cCY  -+,-
  *)*  )()s3   B B' C B$#B$'C ?C CC