̒ge VxddlZddlZddlZddlZddlmZddlmZmZm Z m Z m Z m Z ddl mZddlmZmZddlmZmZmZmZejdZGd d ZGd d eZ ddZdZdZ ddZddZ dZ!dZ"dZ#dZ$ d dZ%dZ&Gddej'Z(dS)!N) has_magic) BaseCache BlockCache BytesCache MMapCacheReadAheadCachecaches)compr) filesystemget_filesystem_class)_unstrip_protocolbuild_name_functioninfer_compressionstringify_pathfsspecc^eZdZdZ d dZdZdZdZdZe d Z d Z d Z dS) OpenFileaF File-like object to be used in a context Can layer (buffered) text-mode and compression over any file-system, which are typically binary-only. These instances are safe to serialize, as the low-level file object is not created until invoked using ``with``. Parameters ---------- fs: FileSystem The file system to use for opening the file. Should be a subclass or duck-type with ``fsspec.spec.AbstractFileSystem`` path: str Location to open mode: str like 'rb', optional Mode of the opened file compression: str or None, optional Compression to apply encoding: str or None, optional The encoding to use if opened in text mode. errors: str or None, optional How to handle encoding errors if opened in text mode. newline: None or str Passed to TextIOWrapper in text mode, how to handle line endings. autoopen: bool If True, calls open() immediately. Mostly used by pickle pos: int If given and autoopen is True, seek to this location immediately rbNc||_||_||_t|||_||_||_||_g|_dSN) fspathmodeget_compression compressionencodingerrorsnewlinefobjects)selfrrrrrrrs +lib/python3.11/site-packages/fsspec/core.py__init__zOpenFile.__init__=sK  *4==     cht|j|j|j|j|j|j|jffSr)rrrrrrrrr!s r" __reduce__zOpenFile.__reduce__Ps8          r$cd|jdS)Nz )rr&s r"__repr__zOpenFile.__repr__^s*TY****r$c|jdddddz}|j|j|}|g|_|j?t|j}|||d}|j|d|jvrz'OpenFiles.__enter__..s ,,,! ,,,r$)r ValueErrorhasattrrRrN)r!rs r"r5zOpenFiles.__enter__s 7?<== = W r;'' "\\$// z!r4   RU%6U -,t,,,,r$c|j}fd|Dd|jvrO t|dr||jdSt|dr|j|j}ndSNdS)Nc$g|] }|j SrI)r9)rTrUr8s r"rVz&OpenFiles.__exit__..s")))qT )))r$r?TrRr)rrrX commit_manyrN)r!r8rs ` r"r9zOpenFiles.__exit__s W))))D)))) di   2{++NN4:...F2t$$):BBE  r$ct|}t|trt ||j|jS|S)NrL)rO __getitem__ isinstanceslicerKrr)r!itemoutrPs r"r]zOpenFiles.__getitem__sJgg!!$'' dE " " >StyTW=== = r$c(dt|dS)Nz )lenr&s r"r)zOpenFiles.__repr__s:3t99::::r$) rDrErFrGr#r5r9r]r) __classcell__rPs@r"rKrKs  $(D       ---    ;;;;;;;r$rKrutf8Tc t|||| || \} } jdkr| _n"dvr| rfd| D}fd|Dtfd| DS)av Given a path or paths, return a list of ``OpenFile`` objects. For writing, a str path must contain the "*" character, which will be filled in by increasing numbers, e.g., "part*" -> "part1", "part2" if num=2. For either reading or writing, can instead provide explicit list of paths. Parameters ---------- urlpath: string or list Absolute or relative filepath(s). Prefix with a protocol like ``s3://`` to read from alternative filesystems. To read from multiple files you can pass a globstring or a list of paths, with the caveat that they must all have the same protocol. mode: 'rb', 'wt', etc. compression: string or None If given, open file using compression codec. Can either be a compression name (a key in ``fsspec.compression.compr``) or "infer" to guess the compression from the filename suffix. encoding: str For text mode only errors: None or str Passed to TextIOWrapper in text mode name_function: function or None if opening a set of files for writing, those files do not yet exist, so we need to generate their names by formatting the urlpath for each sequence number num: int [1] if writing mode, number of files we expect to create (passed to name+function) protocol: str or None If given, overrides the protocol found in the URL. newline: bytes or None Used for line terminator in text mode. If None, uses system default; if blank, uses no translation. auto_mkdir: bool (True) If in write mode, this will ensure the target directory exists before writing, by calling ``fs.mkdirs(exist_ok=True)``. expand: bool **kwargs: dict Extra options that make sense to a particular storage connection, e.g. host, port, username, password, etc. Examples -------- >>> files = open_files('2015-*-*.csv') # doctest: +SKIP >>> files = open_files( ... 's3://bucket/2015-*-*.csv.gz', compression='gzip' ... ) # doctest: +SKIP Returns ------- An ``OpenFiles`` instance, which is a list of ``OpenFile`` objects that can be used as a single context Notes ----- For a full list of the available protocols and the implementations that they map across to see the latest online documentation: - For implementations built into ``fsspec`` see https://filesystem-spec.readthedocs.io/en/latest/api.html#built-in-implementations - For implementations in separate packages see https://filesystem-spec.readthedocs.io/en/latest/api.html#other-known-implementations )num name_functionstorage_optionsprotocolexpandfiler?c:h|]}|SrI)_parent)rTrrs r" zopen_files..$s%6662::d##666r$c>g|]}|dS)T)exist_ok)makedirs)rTparentrs r"rVzopen_files..%s)BBBVd + +BBBr$c <g|]}t|S))rrrrr)r)rTrrrrrrrs r"rVzopen_files..'sO  '!    r$rL)get_fs_token_pathsrk auto_mkdirrK)urlpathrrrrrirhrkrrwrlkwargsfs_tokenpathsparentsrs ```` ` @r" open_filesr}s^- #B% {f" DZ6666666BBBB'BBBB       r$c tjd d|vr! fd|dDn|g}g}d}|}t |D]}|ddpt |dpd}t|}||}||i} ||dur| |td i|| } | |}|dvrd| vr|}| ||| f|}tt |}|S) Nz .*[^a-z]+.*z::cPg|]"}d|vs|r|n|dz#S)://)match)rTpxs r"rVz_un_chain..;s7PPP!eqjjAGGAJJjAIPPPr$rkrrm> filecache blockcache simplecachetarget_protocolrI)recompilesplitcopyr@popsplit_protocolr _get_kwargs_from_urlsupdatedict_strip_protocolr1list) rrybitsra previous_bitbitrkcls extra_kwargskwskwrs @r" _un_chainr8sy =!!A 4<< QPPPtzz$?O?OPPPPV CL [[]]F~~::j$//S>#3F3Fq3ISV"8,,0055 jj2&& $q'>> JJv     ( (L (C ( (!!#&& B B B!++C C2&''' x}}  C Jr$c z hd fd|D}t||}i}tt|D]K\}}|\}}}|t |dz krt di||}0t di|||d<||d<||d<L|d\} }} t |fi|} | | fS) a Turn fully-qualified and potentially chained URL into filesystem instance Parameters ---------- url : str The fsspec-compatible URL **kwargs: dict Extra options that make sense to a particular storage connection, e.g. host, port, username, password, etc. Returns ------- filesystem : FileSystem The new filesystem discovered from ``url`` and created with ``**kwargs``. urlpath : str The file-systems-specific URL for ``url``. >rhrrrlrrrric$i|] \}}|v || SrIrI)rTkv known_kwargss r" zurl_to_fs..ws) G G Gtq!,1F1Fa1F1F1Fr$rtarget_optionsrforrI)itemsr enumerater@rcrr ) urlrychaininkwargsichurlsrkrrx_rrs @r" url_to_fsrWs ,   LH G G Gv||~~ G G GF c6 " "EH8E??++2h E Q  --b-H--H %)%;%;B%;(%;%;!"&."# 8GXq H ) ) ) )B w;r$c `td|g||||||dd|}|st||dS)aGiven a path or paths, return one ``OpenFile`` object. Parameters ---------- urlpath: string or list Absolute or relative filepath. Prefix with a protocol like ``s3://`` to read from alternative filesystems. Should not include glob character(s). mode: 'rb', 'wt', etc. compression: string or None If given, open file using compression codec. Can either be a compression name (a key in ``fsspec.compression.compr``) or "infer" to guess the compression from the filename suffix. encoding: str For text mode only errors: None or str Passed to TextIOWrapper in text mode protocol: str or None If given, overrides the protocol found in the URL. newline: bytes or None Used for line terminator in text mode. If None, uses system default; if blank, uses no translation. **kwargs: dict Extra options that make sense to a particular storage connection, e.g. host, port, username, password, etc. Examples -------- >>> openfile = open('2015-01-01.csv') # doctest: +SKIP >>> openfile = open( ... 's3://bucket/2015-01-01.csv.gz', compression='gzip' ... ) # doctest: +SKIP >>> with openfile as f: ... df = pd.read_csv(f) # doctest: +SKIP ... Returns ------- ``OpenFile`` object. Notes ----- For a full list of the available protocols and the implementations that they map across to see the latest online documentation: - For implementations built into ``fsspec`` see https://filesystem-spec.readthedocs.io/en/latest/api.html#built-in-implementations - For implementations in separate packages see https://filesystem-spec.readthedocs.io/en/latest/api.html#other-known-implementations F)rxrrrrrkrrlrrI)r}FileNotFoundError) rxrrrrrkrryras r"r0r0scx        C )((( q6Mr$c Dd|vrtdt|fd|i|}t|djddstd|5}d|D}d d d n #1swxYwYt |t rt |s|dS|S) aKOpen file(s) which can be resolved to local For files which either are local, or get downloaded upon open (e.g., by file caching) Parameters ---------- url: str or list(str) mode: str Must be read mode storage_options: passed on to FS for or used by open_files (e.g., compression) r?z(Can only ensure local files when readingrr local_fileFzOopen_local can only be used on a filesystem which has attribute local_file=Truecg|] }|j SrI)name)rTr3s r"rVzopen_local..s'''A'''r$N)rWr}getattrrr^strr)rrrjofrNr{s r" open_localrs $CDDD C 6 6d 6o 6 6B 2a58\5 1 1  -    (u'''''(((((((((((((((#sIcNNQx Ls A))A-0A-cl|dkrt|}||tvrtd|d|S)NinferzCompression type z not supported)rr rW)rxrs r"rrsJg'00 ;e#;#;H[HHHIII r$ct|}d|vr0|dd\}}t|dkr||fSd|fS)zReturn protocol, path pairrrN)rrrc)rxrkrs r"rrsVW%%G  ua00$ x==1  T> ! =r$cnt|\}}t|}||S)zCReturn only path part of full URL, according to appropriate backend)rr r)rxrkrrs r"strip_protocolrs4 ))KHa x ( (C   w ' ''r$c(g}t|}d|vrtd|Ddkrtdt|t |}|D]@}d|vr%|t |||+||At ||kr |d|}nR|D]O}t|r)|| |:||P|S)a(Expand paths if they have a ``*`` in them (write mode) or any of ``*?[]`` in them (read mode). :param paths: list of paths mode: str Mode in which to open files. num: int If opening in writing mode, number of files we expect to create. fs: filesystem object name_function: callable If opening in writing mode, this callable is used to generate path names. Names are generated for each partition by ``urlpath.replace('*', name_function(partition_index))``. :return: list of paths wcg|]}d|vd S)*rrI)rTrs r"rVz*expand_paths_if_needed..s---aC1HHHHHr$rz;When writing data, only one filename mask can be specified.rN) rsumrWmaxrcextend _expand_pathsr1rglob)r{rrhrriexpanded_paths curr_paths r"expand_paths_if_neededr sB N KKE d{{ --5--- . . 2 2M #s5zz"" 1 1Ii%%mI}c&R&RSSSS%%i0000 ~   $ $+DSD1N 1 1I## 1%%bggi&8&89999%%i0000 r$cZt|tttfr4|st dt t|d}nt |}pi|r|d<t |pi}i} tt|D]K\} } | \} } }| t|dz krtdi|| } 0tdi|| | d<| | d<| | d<L|d\}}}t|fi| t|tttfrHfd|D}td |Ddkrt d |d |D}n |}t|tttfrt||||}nfd |vr|rt|||}nNd |vr|rt|||}n6d|vr/fdt|D}n|g}j|fS)aFilesystem, deterministic token, and paths from a urlpath and options. Parameters ---------- urlpath: string or iterable Absolute or relative filepath, URL (may include protocols like ``s3://``), or globstring pointing to data. mode: str, optional Mode in which to open files. num: int, optional If opening in writing mode, number of files we expect to create. name_function: callable, optional If opening in writing mode, this callable is used to generate path names. Names are generated for each partition by ``urlpath.replace('*', name_function(partition_index))``. storage_options: dict, optional Additional keywords to pass to the filesystem class. protocol: str or None To override the protocol specifier in the URL expand: bool Expand string paths for writing, assuming the path is a directory zempty urlpath sequencerrkrrrrcZg|]'}tt|pid(Sr)rr)rTurjs r"rVz&get_fs_token_paths..psB   GHInQ'')>B ? ? B   r$ch|] }|d S)rrIrTpcs r"rpz%get_fs_token_paths..ss((("1(((r$z$Protocol mismatch getting fs from %scg|] }|d SrrIrs r"rVz&get_fs_token_paths..us)))2A)))r$rrrc>g|]}||SrI)isdir)rTr3rs r"rVz&get_fs_token_paths..s(JJJ1bhhqkkJQJJJr$rI)r^rtuplesetrWrrrr@rcrr rrrsortedr _fs_token)rxrrhrirjrkrlurlpath0rrrrrnested_protocolrr{rpchainsrs ` @r"rvrv:s>'D%-..+ 7566 6!$w--"233!'**%+O/&. # h 52 6 6EH8E??++2$&!or E Q  --b-H--H %)%;%;B%;(%;%;!"&5"#qE8Q H ) ) ) )B'D%-..*    LS    ((((( ) )A - -CWMM M)))))""5))%$s+,, &udC]KK $;;6;!%<.s1IIIc==#3#344IIIr$zqIn order to preserve order between partitions paths created with ``name_function`` should sort to partition orderzPath should be either 1. A list of paths: ['foo.json', 'bar.json', ...] 2. A directory: 'foo/ 3. A path with a '*' in it: 'foo.*.json')r^rcountrWosrjoinrrangerloggerwarningrrrc)rrirhr{s`` r"rrs$ ::c??Q  MNN N __7<<h//D  /a88MIIIIIeCjjIII F5MM ! ! NN*    D5$- ( (  4yyCT  7   Lr$c4eZdZdZ dfd ZdZxZS)r2zTextIOWrapper cannot be pickled. This solves it. Requires that ``buffer`` be pickleable, which all instances of AbstractBufferedFile are. NFcX||||||f|_tj|jdSr)r8rOr#)r!bufferrrrline_buffering write_throughrPs r"r#z PickleableTextIOWrapper.__init__s5Hfg~}T $)$$$$r$ct|jfSr)r2r8r&s r"r'z"PickleableTextIOWrapper.__reduce__s& 11r$)NNNFF)rDrErFrGr#r'rdres@r"r2r2sg % % % % % %2222222r$r2) rNrfNNrNNTT)rNrfNNN)r)rrNNNT))iologgingrrrrcachingrrrrr r rr registryr r utilsrrrr getLoggerrrrrKr}rrr0rrrrrrvr TextIOWrapperr2rIr$r"rs  66666666  8 $ $ppppppppf7;7;7;7;7;7;7;7;x      llll^>...f     IIIIX:(((,,,b    J#J#J#J#Z<22222b.22222r$