̒gemp UddlmZddlZddlZddlZddlZddlZddlZddlZddl m Z m Z ddl m Z mZmZmZmZmZmZmZe rddlZddlmZedZn edZedZejdZeeegefZGd d ZGd d eZ Gd deZ!GddeZ"GddeZ#GddeZ$GddeZ%GddeZ&GddeeefZ'GddeZ(deiZ)de*d<d'd(d&Z+ee e$e!e#e"e%e&e(f D] Z,e+e,dS))) annotationsN)FutureThreadPoolExecutor) TYPE_CHECKINGAnyCallableClassVarGeneric NamedTuple OrderedDictTypeVar) ParamSpecPTfsspecc2eZdZUdZdZded<dd ZddZdS) BaseCacheagPass-though cache: doesn't keep anything, calls every time Acts as base class for other cachers Parameters ---------- blocksize: int How far to read ahead in numbers of bytes fetcher: func Function of the form f(start, end) which gets bytes from remote as specified size: int How big this file is none ClassVar[str]name blocksizeintfetcherFetchersizereturnNonec0||_||_||_dSN)rrr)selfrrrs .lib/python3.11/site-packages/fsspec/caching.py__init__zBaseCache.__init__9s"  start int | Nonestopbytescn|d}||j}||jks||krdS|||SNrr#)rrr r$r&s r!_fetchzBaseCache._fetch>sF =E <9D DI  $3||E4(((r#Nrrrrrrrrr$r%r&r%rr')__name__ __module__ __qualname____doc__r__annotations__r"r+r#r!rr's[  !D     ))))))r#rcNeZdZdZdZ ddfd ZddZddZddZd dZ xZ S)! MMapCachezmemory-mapped sparse file cache Opens temporary file, which is filled blocks-wise when data is requested. Ensure there is enough disc space in the temporary location. This cache method might only work on posix mmapNrrrrrlocation str | Noneblocksset[int] | Nonerrct||||tn||_||_||_dSr)superr"setr9r7 _makefilecache)r rrrr7r9 __class__s r!r"zMMapCache.__init__SsP GT222%~ceee6   ^^%% r#mmap.mmap | bytearrayc*ddl}ddl}|jdkrtS|j$t j|js|j(|}t|_ nt|jd}| |jdz | d|nt|jd}|j||jS)Nrzwb+1zrb+)r6tempfiler bytearrayr7ospathexists TemporaryFiler=r9openseekwriteflushfileno)r r6rEfds r!r>zMMapCache._makefile`s  9>>;;  = t}(E(E }$++--!ee $-// GGDIM " " " HHTNNN HHJJJJdmU++Btydi000r#r$r%endr'cBtd|d||d}|j}|jks||krdS|jz}|jz}fdt ||dzD}|r|d}|jz}t |jzj}td|d|d|||j||<j ||j||S) NzMMap cache fetching -rr#c&g|] }|jv |Sr3)r9).0ir s r! z$MMapCache._fetch..s%UUUa@T@T@T@T@Tr#rCzMMap get block #z () loggerdebugrrrangepopminrr?r9add) r r$rQ start_block end_blockneedrVsstartsends ` r!r+zMMapCache._fetchvsL 9E99C99::: =E ;)C DI  #3t~- 4>) UUUU5i!m<<UUU  A'Fv. ::D LL@A@@@@$@@ A A A&*ll64&@&@DJvd{ # KOOA    z%)$$r#dict[str, Any]c>|j}|d=|S)Nr?)__dict__copyr states r! __getstate__zMMapCache.__getstate__s! ""$$ 'N r#rhcl|j|||_dSr)reupdater>r?rgs r! __setstate__zMMapCache.__setstate__s- U###^^%% r#)NN) rrrrrrr7r8r9r:rr)rrAr$r%rQr%rr'rrcrhrcrr) r.r/r0r1rr"r>r+rirl __classcell__r@s@r!r5r5Hs D $"& & & & & & & &1111,%%%%. &&&&&&&&r#r5c0eZdZdZdZdfd ZddZxZS)ReadAheadCachea!Cache which reads only when we get beyond a block of data This is a much simpler version of BytesCache, and does not attempt to fill holes in the cache or keep fragments alive. It is best suited to many small reads in a sequential order (e.g., reading lines from a file). readaheadrrrrrrrcxt|||d|_d|_d|_dS)Nr#r)r<r"r?r$rQr rrrr@s r!r"zReadAheadCache.__init__s8 GT222  r#r$r%rQr'cJ|d}| ||jkr|j}||jks||krdS||z }||jkr*||jkr|j||jz ||jz S|j|cxkr |jkr4nn1|j||jz d}|t |z}|j}nd}t |j||jz}||||_||_|jt |jz|_||jd|zSr))rr$rQr?lenr\rr)r r$rQlparts r!r+zReadAheadCache._fetchs9 =E ;# //)C DI  #3 %K DJ  3$(??:edj033CCD D Z5 + + + +48 + + + + +:edj0223D TNAHEED$)S4>122\\%--  :DJ/dj!n$$r#r,rmr.r/r0r1rr"r+rprqs@r!rsrssb D %%%%%%%%r#rsc0eZdZdZdZdfd ZddZxZS)FirstChunkCachezCaches the first block of a file only This may be useful for file types where the metadata is stored in the header, but is randomly accessed. firstrrrrrrrc\t|||d|_dSr)r<r"r?rvs r!r"zFirstChunkCache.__init__s* GT222#' r#r$r%rQr'c|pd}|p|j}||jkr|j_||jkr4|d|}|d|j|_||dS|d|j|_|j||}||jkr|||j|z }|S|||SNr)rrr?r)r r$rQdatarzs r!r+zFirstChunkCache._fetchs TY 4> ! !z!''<<3//D!%&6&6!7DJ<'!\\!T^<< :eCi(DT^## T^S999K<<s++ +r#r,rmr{rqs@r!r}r}sb D((((((,,,,,,,,r#r}cfeZdZdZdZ ddfd Zd dZdZd!dZd"dZ d#dZ d$fd Z d%dZ xZ S)& BlockCachea Cache holding memory as a set of blocks. Requests are only ever made ``blocksize`` at a time, and are stored in an LRU cache. The least recently accessed block is discarded when more than ``maxblocks`` are stored. Parameters ---------- blocksize : int The number of bytes to store in each block. Requests are only ever made for ``blocksize``, so this should balance the overhead of making a request against the granularity of the blocks. fetcher : Callable size : int The total size of the file being cached. maxblocks : int The maximum number of blocks to cache for. The maximum memory use for this cache is then ``blocksize * maxblocks``. blockcache rrrrr maxblocksrrct|||tj||z |_||_t j||j|_ dSr) r<r"mathceilnblocksr functools lru_cache _fetch_block_fetch_block_cachedr rrrrr@s r!r"zBlockCache.__init__sd GT222y !122 "#A9#6y#A#A$BS#T#T   r#strc8d|jd|jd|jdS)Nzrrrr s r!__repr__zBlockCache.__repr__s= 9T^ 9 9I 9 9)- 9 9 9 r#c4|jSz The statistics on the block cache. Returns ------- NamedTuple Returned directly from the LRU Cache used internally. r cache_infors r!rzBlockCache.cache_info '22444r#rcc|j}|d=|S)Nrrergs r!rizBlockCache.__getstate__s  ' ( r#rhc|j|tj|d|j|_dS)Nr)rerkrrrrrgs r!rlzBlockCache.__setstate__sH U####J9#6u[7I#J#J  $ $    r#r$r%rQr'c|d}||j}||jks||krdS||jz}||jz}t||dzD]}|||||||S)Nrr#rCstart_block_numberend_block_number)rrrZr _read_cache)r r$rQrr block_numbers r!r+zBlockCache._fetchs =E ;)C DI  #3#dn4$.0""46F6JKK 3 3L  $ $\ 2 2 2 2  1-     r#rc||jkrtd|d|jd||jz}||jz}td|t ||}|S)= Fetch the block of data for `block_number`. 'block_number=(' is greater than the number of blocks ()zBlockCache fetching block %dr ValueErrorrrXinfor<r+)r rr$rQblock_contentsr@s r!rzBlockCache._fetch_block6s $, & &999)-999  t~-dn$ 2LAAAs33r#rrc||jz}||jz}||kr||}|||Sg}||||dt|dz|D]*} ||| +|||d|d|Sz Read from our block cache. Parameters ---------- start, end : int The start and end byte positions. start_block_number, end_block_number : int The start and end block numbers. NrCr#rrappendrZjoin r r$rQrr start_posend_posblockoutrs r!rzBlockCache._read_cacheFsDN* & !1 1 1334FGGE7*+ +C JJt//0BCCIJJO P P P !&&81&<>N O O C C  433LAABBBB JJt//0@AA(7(K L L L88C== r#r rrrrrrrrrrrrrnrorm)rrrr' r$rrQrrrrrrr')r.r/r0r1rr"rrrirlr+rrrprqs@r!rrs, DMOUUUUUUU    5 5 5         . "!"!"!"!"!"!"!"!r#rcHeZdZUdZdZded< ddfd ZddZddZxZ S) BytesCacheaKCache which holds data in a in-memory bytes object Implements read-ahead by the block size, for semi-random reads progressing through the file. Parameters ---------- trim: bool As we read more data, whether to discard the start of the buffer when we are more than a blocksize ahead of it. r'rrTrrrrrtrimboolrrct|||d|_d|_d|_||_dS)Nr#)r<r"r?r$rQr)r rrrrr@s r!r"zBytesCache.__init__zsA GT222 !% # r#r$r%rQc$|d}||j}||jks||krdS|j<||jkr1|j*||jkr||jz }|j|||z|z S|jrt |j||jz}n|}||ks ||jkrdS|j ||jkr6|j ||jkr$||||_||_n|jJ|jJ||jkro|j|j|z |jkr#||||_||_n|||j}||_||jz|_n|j|||jkrq|j|jkrn`||jz |jkr#||||_||_n*||j|}|j|z|_|jt|jz|_||jz }|j|||z|z }|jrT|j|jz |jdzz}|dkr4|xj|j|zz c_|j|j|zd|_|S)Nrr#rC) rr$rQr?rr\rrxr)r r$rQoffsetbendnewrnums r!r+zBytesCache._fetchs =E ;)C DI  #3 J "##$dhTZ'F:fv|e';;< < > ty#"677DDD 5==EDI--3 J %$*"4"4 H dheT22DJDJJ:)))8'''tz!!8#tx#~'F'F!%eT!:!:DJ!&DJJ,,udj99C!&DJ!$tz!1DJJ%$//8di''48^dn44!%eT!:!:DJ!&DJJ,,tx66C!%c!1DJ:DJ/#j&3,"667 9 @8dj(dnq.@ACQww dns22 !Z(<(>(>?  r#c*t|jSr)rxr?rs r!__len__zBytesCache.__len__s4:r#)T) rrrrrrrrrrrm)rr) r.r/r0r1rr2r"r+rrprqs@r!rrks  "D!!!!IM????Br#rcFeZdZUdZdZded< ddfd ZddZxZS)AllBytesz!Cache entire contents of the fileallrrNrr%rFetcher | Nonerr bytes | Nonerrct|||||d|j}||_dSr)r<r"rrr)r rrrrr@s r!r"zAllBytes.__init__sF GT222 <<<49--D r#r$r&r'c |j||Sr)rr*s r!r+zAllBytes._fetchsyt$$r#)NNNN) rr%rrrr%rrrrr- r.r/r0r1rr2r"r+rprqs@r!rrs}++D!%"&!       %%%%%%%%r#rcFeZdZUdZdZded<idfdfd Zdfd ZxZS)KnownPartsOfAFilea Cache holding known file parts. Parameters ---------- blocksize: int How far to read ahead in numbers of bytes fetcher: func Function of the form f(start, end) which gets bytes from remote as specified size: int How big this file is data: dict A dictionary mapping explicit `(start, stop)` file-offset tuples with known bytes. strict: bool, default True Whether to fetch reads that go beyond a known byte-range boundary. If `False`, any read that ends outside a known part will be zero padded. Note that zero padding will not be used for reads that begin outside a known byte-range. partsrrTrrrrrrdict[tuple[int, int], bytes]strictr_rc `t|||||_|rt|}|dg}||dg} |ddD]\} } |d\} } | | kr-| | f|d<| dxx|| | fz cc<C|| | f| || | ftt|| |_ dS||_ dS)NrrC) r<r"rsortedkeysr[rdictzipr)r rrrrrr old_offsetsoffsetsr9r$r&start0stop0r@s r!r"zKnownPartsOfAFile.__init__s4 GT222    --K"1~&Ghh{1~../F*122 ; ; t ' E>>#)4.GBK2JJJ$((E4="9"99JJJJNNE4=111MM$((E4="9"9::::S&1122DIIIDIIIr#r$r%r&rr'c|d}||j}d}|jD]g\\}}}||cxkr|krRn||z }||||z|z }|jr||cxkr|kr"nn|d||z t |z zz }|cS|}nh|jt d||fdtjd||fdt d|d||t ||zS) Nrr#z&Read is outside the known file parts: z. z%. IO/caching performance may be poor!z!KnownPartsOfAFile cache fetching rS) rritemsrrxrrwarningswarnrXrYr<r+) r r$r&rloc0loc1roffr@s r!r+zKnownPartsOfAFile._fetchs =E <9D"&)//"3"3   LT4$u####t#####dl3te!334{ dd&:&:&:&:d&:&:&:&:&: 7dUlSXX&=>>CJJJ !EE$( < WudmWWWXX X   3eT] 3 3 3     GGGGGHHHUWW^^E40000r#) rrrrrrrrrrrrr-rrqs@r!rrs,"D!!!!.0 <(1(1(1(1(1(1(1(1(1(1r#rcXeZdZdZGddeZddd ZddZddZddZ ddZ dS) UpdatableLRUzg Custom implementation of LRU cache that allows updating keys Used by BackgroudBlockCache c8eZdZUded<ded<ded<ded<dS)UpdatableLRU.CacheInforhitsmissesmaxsizecurrsizeN)r.r/r0r2r3r#r! CacheInforFs4    r#rfuncCallable[P, T]max_sizerrrctj|_||_||_d|_d|_tj|_ dSr) collectionsr _cache_func _max_size_hits_misses threadingLock_lock)r rrs r!r"zUpdatableLRU.__init__LsA+6+B+D+D  !  ^%% r#argsP.argskwargsP.kwargsrc&|r$td||j5||jvrC|j||xjdz c_|j|cdddS dddn #1swxYwY|j|i|}|j5||j|<|xjdz c_t|j|j kr|j ddddn #1swxYwY|S)Nz Got unexpected keyword argument rCFlast) TypeErrorrrr move_to_endrrrrxrpopitem)r rrresults r!__call__zUpdatableLRU.__call__Ts  PNv{{}}NNOO O Z ) )t{"" ''--- a {4(  ) ) ) ) ) ) ) )" ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) T,V,, Z 0 0 &DK  LLA LL4;$.00 ###///  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0  s%ABB B 'ADD  D rrcV|j5||jvcdddS#1swxYwYdSr)rr)r rs r! is_key_cachedzUpdatableLRU.is_key_cachedgsx Z ' '4;& ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' 's ""r c|j5||j|<t|j|jkr|jdddddS#1swxYwYdS)NFr)rrrxrr )r r rs r!add_keyzUpdatableLRU.add_keyks Z 0 0 &DK 4;$.00 ###/// 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0sAAAArc|j5||jt|j|j|jcdddS#1swxYwYdS)N)rrrr)rrrrxrrrrs r!rzUpdatableLRU.cache_infoqs Z  >>T[))Z| "                  s:AAAN)r)rrrrrr)rrrrrr)rrrr)r rrrrrrr) r.r/r0r1r rr"r rrrr3r#r!rr?s J &&&&&&''''0000 r#rcveZdZUdZdZded< d"d#fd Zd$dZd%dZd&dZ d'dZ d(dZ d)d*fd Z d+d!Z xZS),BackgroundBlockCachea Cache holding memory as a set of blocks with pre-loading of the next block in the background. Requests are only ever made ``blocksize`` at a time, and are stored in an LRU cache. The least recently accessed block is discarded when more than ``maxblocks`` are stored. If the next block is not in cache, it is loaded in a separate thread in non-blocking way. Parameters ---------- blocksize : int The number of bytes to store in each block. Requests are only ever made for ``blocksize``, so this should balance the overhead of making a request against the granularity of the blocks. fetcher : Callable size : int The total size of the file being cached. maxblocks : int The maximum number of blocks to cache for. The maximum memory use for this cache is then ``blocksize * maxblocks``. backgroundrrrrrrrrrrrc>t|||tj||z |_||_t |j||_td|_ d|_ d|_ tj|_dS)NrC max_workers)r<r"rrrrrrrr_thread_executor_fetch_future_block_number _fetch_futurerr_fetch_future_lockrs r!r"zBackgroundBlockCache.__init__s GT222y !122 "#/0A9#M#M 2q A A A6:'37"+."2"2r#rc8d|jd|jd|jdS)Nz >?OPP53C/%)%:%A%A%'7&&"                 1-     s&C DDD!AHH H syncrlog_infoc||jkrtd|d|jd||jz}||jz}td||t ||}|S)rrrrz!BlockCache fetching block (%s) %dr)r rr*r$rQrr@s r!rz!BackgroundBlockCache._fetch_blocks $, & &999)-999  t~-dn$ 7<PPPs33r#rrc||jz}||jz}||kr||}|||Sg}||||dt|dz|D]*} ||| +|||d|d|Srrrs r!rz BackgroundBlockCache._read_cache"sDN* & !1 1 1,,-?@@E7*+ +C JJt//0BCCIJJO P P P !&&81&<>N O O C C  433LAABBBB JJt//0@AA(7(K L L L88C== r#rrrrrn)rrrm)r))rrr*rrr'r)r.r/r0r1rr2r"rrrirlr+rrrprqs@r!rr{s2'D&&&&MO 3 3 3 3 3 3 3    5 5 5 53333J J J J X "!"!"!"!"!"!"!"!r#rz!dict[str | None, type[BaseCache]]cachesFclstype[BaseCache]clobberrrrc~|j}|s)|tvr td|dt||t|<dS)z'Register' cache implementation. Parameters ---------- clobber: bool, optional If set to True (default is False) - allow to overwrite existing entry. Raises ------ ValueError zCache with name z is already known: N)rr-r)r.r0rs r!register_cacher2MsL 8D Wtv~~UDUUvd|UUVVVF4LLLr#)F)r.r/r0rrr)- __future__rrrloggingrrGrrconcurrent.futuresrrtypingrrrr r r r r r6typing_extensionsrrr getLoggerrXrr'rrr5rsr}rrrrrrr-r2r2cr3r#r!r:s""""""" 99999999                    KKK++++++ #AA A GCLL  8 $ $ C:u$ %))))))))BN&N&N&N&N& N&N&N&b'%'%'%'%'%Y'%'%'%T,,,,,i,,,>F!F!F!F!F!F!F!F!RZZZZZZZZz%%%%%y%%%*_1_1_1_1_1 _1_1_1D9999971a4=999xI!I!I!I!I!9I!I!I!\ )- (     AN1  r#