Yh UddlmZddlZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z m Z ddl mZddlmZddlmZmZmZmZmZmZmZerddlZddlmZed Zn ed Zed Zejd Zee e ge!fZ"ee#e e fge!fZ$Gd d Z%Gdde%Z&Gdde%Z'Gdde%Z(Gdde%Z)Gdde%Z*Gdde%Z+Gdde%Z,GddeeefZ-Gdde%Z.de%iZ/d e0d!<d*d+d)Z1e%e&e*e'e)e(e+e,e.f D] Z2e1e2dS),) annotationsN) OrderedDict)FutureThreadPoolExecutor)groupby) itemgetter) TYPE_CHECKINGAnyCallableClassVarGeneric NamedTupleTypeVar) ParamSpecPTfsspeccJeZdZUdZdZded<dd ZddZddZddZ ddZ dS) BaseCacheagPass-though cache: doesn't keep anything, calls every time Acts as base class for other cachers Parameters ---------- blocksize: int How far to read ahead in numbers of bytes fetcher: func Function of the form f(start, end) which gets bytes from remote as specified size: int How big this file is none ClassVar[str]name blocksizeintfetcherFetchersizereturnNonech||_d|_||_||_d|_d|_d|_dSNr)rnblocksrr hit_count miss_counttotal_requested_bytes)selfrrrs `/var/www/tools.fuzzalab.pt/emblema-extractor/venv/lib/python3.11/site-packages/fsspec/caching.py__init__zBaseCache.__init__<s:"   %&"""start int | Nonestopbytescn|d}||j}||jks||krdS|||S)Nrr))rrr&r*r,s r'_fetchzBaseCache._fetchFsF =E <9D DI  $3||E4(((r)c0d|_d|_d|_dS)zAReset hit and miss counts for a more ganular report e.g. by file.rN)r#r$r%r&s r' _reset_statszBaseCache._reset_statsOs%&"""r)strc x|jdkr |jdkrdSd|jd|jd|jd|jd S)z2Return a formatted string of the cache statistics.rz , z: z hits, z misses, z total requested bytes)r#r$rr%r2s r' _log_statszBaseCache._log_statsUst >Q  4?a#7#72FTYFF$.FFFFSWSmFFF Fr)cd|jjd|jd|jd|jd|jd|jd|jdS) Nz ) __class____name__rr"rr#r$r%r2s r'__repr__zBaseCache.__repr__\s . !  !^  "\  "Y   "^   "_   %)$>    r)Nrrrrrrrrr*r+r,r+rr-rr)rr4) r: __module__ __qualname____doc__r__annotations__r(r0r3r7r;r)r'rr*s  !D    ''''))))'''' FFFF       r)rcPeZdZdZdZ ddfd ZddZd dZd!dZd"dZ xZ S)# MMapCacheamemory-mapped sparse file cache Opens temporary file, which is filled blocks-wise when data is requested. Ensure there is enough disc space in the temporary location. This cache method might only work on posix Parameters ---------- blocksize: int How far to read ahead in numbers of bytes fetcher: Fetcher Function of the form f(start, end) which gets bytes from remote as specified size: int How big this file is location: str Where to create the temporary file. If None, a temporary file is created using tempfile.TemporaryFile(). blocks: set[int] Set of block numbers that have already been fetched. If None, an empty set is created. multi_fetcher: MultiFetcher Function of the form f([(start, end)]) which gets bytes from remote as specified. This function is used to fetch multiple blocks at once. If not specified, the fetcher function is used instead. mmapNrrrrrlocation str | Noneblocksset[int] | None multi_fetcherMultiFetcher | Nonerrct||||tn||_||_||_||_dSN)superr(setrIrGrK _makefilecache)r&rrrrGrIrKr9s r'r(zMMapCache.__init__sX GT222%~ceee6   *^^%% r)mmap.mmap | bytearrayc*ddl}ddl}|jdkrtS|j$t j|js|j(|}t|_ nt|jd}| |jdz | d|nt|jd}|j||jS)Nrzwb+1zr+b)rFtempfiler bytearrayrGospathexists TemporaryFilerPrIopenseekwriteflushfileno)r&rFrWfds r'rQzMMapCache._makefiles  9>>;;  = t}(E(E }$++--!ee $-// GGDIM " " " HHTNNN HHJJJJdmU++Btydi000r)r*r+endr-c Xtd|d||d}|j}|jks||krdS|jz}|jz}t ||dz}fd|D}xjt fd|Dz c_g}tt|d D]\}} tttd| } | djz} t| d jzjzj} xj | | z z c_ td | dd| d d | d| d || | fj| xjt%| z c_|sj||Sjr}td |t|D]<\} } || \} } td| d| | j| | <=nH|D]E\} } td| d| | | j| | <Fj||S)NzMMap cache fetching -rr)rUc3.K|]}|jv |VdSrNrI.0ir&s r' z#MMapCache._fetch..s/??a!4;*>*>*>*>*>*>??r)c3.K|]}|jv dVdS)rUNrgrhs r'rkz#MMapCache._fetch..s/IIAT[8H8Ha8H8H8H8HIIr)c$|d|dz S)NrrUrC)xs r'z"MMapCache._fetch..s1!r))keyzMMap get blocks z ()zMMap copy block (zMMap get block ()loggerdebugrrranger#sumr enumeratetuplemaprminr%appendrIupdater$lenrRrKr)r&r*rc start_block end_block block_rangeneedranges__blockssstartsendidxrs` r'r0zMMapCache._fetchs  9E99C99::: =E ;)C DI  #3t~- 4>) KQ77 ????;??? #IIIIIIIIII")D//7L7LMMM , ,JAwC 1 w7788GQZ$.0Fwr{T^3dnDdiPPD  & &$- 7 & & LLO71:OO OOvOOOOO    MM64. ) ) ) K  w ' ' ' OOs7|| +OOO ):eCi( (   E LL4F44 5 5 5#D$6$6v$>$>?? , ,Q!' @@@$@@AAA*+ 6$;'' , !' E E  ?????@@@*.,,vt*D*D 6$;''z%)$$r)dict[str, Any]c>|j}|d=|S)NrR)__dict__copyr&states r' __getstate__zMMapCache.__getstate__s! ""$$ 'N r)rcl|j|||_dSrN)rr|rQrRrs r' __setstate__zMMapCache.__setstate__s- U###^^%% r))NNN)rrrrrrrGrHrIrJrKrLrr)rrSr*r+rcr+rr-rrrrrr) r:r?r@rArr(rQr0rr __classcell__r9s@r'rErEis8 D $"&-1 & & & & & & &1111,<%<%<%<%| &&&&&&&&r)rEc0eZdZdZdZdfd ZddZxZS)ReadAheadCachea!Cache which reads only when we get beyond a block of data This is a much simpler version of BytesCache, and does not attempt to fill holes in the cache or keep fragments alive. It is best suited to many small reads in a sequential order (e.g., reading lines from a file). readaheadrrrrrrrcxt|||d|_d|_d|_dS)Nr)r)rOr(rRr*rcr&rrrr9s r'r(zReadAheadCache.__init__s8 GT222  r)r*r+rcr-c|d}| ||jkr|j}||jks||krdS||z }||jkr:||jkr/|xjdz c_|j||jz ||jz S|j|cxkr |jkrDnnA|xjdz c_|j||jz d}|t |z}|j}n|xjdz c_d}t|j||jz}|xj ||z z c_ | |||_||_|jt |jz|_||jd|zSNrr)rU) rr*rcr#rRr$r}rzrr%r)r&r*rclparts r'r0zReadAheadCache._fetchs =E ;# //)C DI  #3 %K DJ  3$(?? NNa NN:edj033CCD D Z5 + + + +48 + + + + + OOq OO:edj0223D TNAHEE OOq OOD$)S4>122 ""cEk1""\\%--  :DJ/dj!n$$r)r<rr:r?r@rArr(r0rrs@r'rrsb D %%%%%%%%r)rc0eZdZdZdZdfd ZddZxZS)FirstChunkCachezCaches the first block of a file only This may be useful for file types where the metadata is stored in the header, but is randomly accessed. firstrrrrrrrcl||kr|}t|||d|_dSrN)rOr(rRrs r'r(zFirstChunkCache.__init__.s: t  I GT222#' r)r*r+rcr-c|pd}||jkrtddSt||j}||jkr|j|xjdz c_||jkrD|xj|z c_|d|}|d|j|_||dS|d|j|_|xj|jz c_|j||}||jkr6|xj||jz z c_|||j|z }|xj dz c_ |S|xjdz c_|xj||z z c_|||S)Nrz,FirstChunkCache: requested start > file sizer)rU) rrsrtrzrrRr$r%rr#)r&r*rcdatars r'r0zFirstChunkCache._fetch5s  49   LLG H H H3#ty!! 4> ! !z!1$''..#5..<<3//D!%&6&6!7DJ<'!\\!T^<< **dn<**:eCi(DT^##**cDN.BB** T^S999 NNa NNK OOq OO  & &#+ 5 & &<<s++ +r)r<rrrs@r'rr%sb D((((((,,,,,,,,r)rc^eZdZdZdZ ddfd Zd ZddZddZd dZ d!fd Z d"dZ xZ S)# BlockCachea Cache holding memory as a set of blocks. Requests are only ever made ``blocksize`` at a time, and are stored in an LRU cache. The least recently accessed block is discarded when more than ``maxblocks`` are stored. Parameters ---------- blocksize : int The number of bytes to store in each block. Requests are only ever made for ``blocksize``, so this should balance the overhead of making a request against the granularity of the blocks. fetcher : Callable size : int The total size of the file being cached. maxblocks : int The maximum number of blocks to cache for. The maximum memory use for this cache is then ``blocksize * maxblocks``. blockcache rrrrr maxblocksrrct|||tj||z |_||_t j||j|_ dSrN) rOr(mathceilr"r functools lru_cache _fetch_block_fetch_block_cachedr&rrrrr9s r'r(zBlockCache.__init__lsd GT222y !122 "#A9#6y#A#A$BS#T#T   r)c4|jSz The statistics on the block cache. Returns ------- NamedTuple Returned directly from the LRU Cache used internally. r cache_infor2s r'rzBlockCache.cache_infot'22444r)rc|j}|d=|S)Nrrrs r'rzBlockCache.__getstate__s  ' ( r)rc|j|tj|d|j|_dS)Nr)rr|rrrrrs r'rzBlockCache.__setstate__sH U####J9#6u[7I#J#J  $ $    r)r*r+rcr-c|d}||j}||jks||krdS||jz}||jz}t||dzD]}|||||||S)Nrr)rUstart_block_numberend_block_number)rrrur _read_cache)r&r*rcrr block_numbers r'r0zBlockCache._fetchs =E ;)C DI  #3#dn4$.0""46F6JKK 3 3L  $ $\ 2 2 2 2  1-     r)rc<||jkrtd|d|jd||jz}||jz}|xj||z z c_|xjdz c_t d|t||}|S)= Fetch the block of data for `block_number`. 'block_number=(' is greater than the number of blocks (rrrUzBlockCache fetching block %d) r" ValueErrorrr%r$rsinforOr0)r&rr*rcblock_contentsr9s r'rzBlockCache._fetch_blocks $, & &999)-999  t~-dn$ ""cEk1"" 1 2LAAAs33r)rrc ||jz}||jz}|xjdz c_||kr||}|||S|||dg}|t |jt |dz||||d|d|Sz Read from our block cache. Parameters ---------- start, end : int The start and end byte positions. start_block_number, end_block_number : int The start and end block numbers. rUNr)rr#rextendryrur{join r&r*rcrr start_posend_posblockouts r'rzBlockCache._read_cachesDN* & ! !1 1 1334FGGE7*+ +++,>?? KLC JJ,,q02BCC    JJt//0@AA(7(K L L L88C== r)r rrrrrrrrrrrrr)rrrr- r*rrcrrrrrrr-) r:r?r@rArr(rrrr0rrrrs@r'rrSs, DMOUUUUUUU 5 5 5         .$&!&!&!&!&!&!&!&!r)rcHeZdZUdZdZded< ddfd ZddZddZxZ S) BytesCacheaKCache which holds data in a in-memory bytes object Implements read-ahead by the block size, for semi-random reads progressing through the file. Parameters ---------- trim: bool As we read more data, whether to discard the start of the buffer when we are more than a blocksize ahead of it. r-rrTrrrrrtrimboolrrct|||d|_d|_d|_||_dS)Nr))rOr(rRr*rcr)r&rrrrr9s r'r(zBytesCache.__init__sA GT222 !% # r)r*r+rccV|d}||j}||jks||krdS|jL||jkrA|j:||jkr/||jz }|xjdz c_|j|||z|z S|jrt |j||jz}n|}||ks ||jkrdS|j ||jkrY|j ||jkrG|xj||z z c_|xjdz c_| |||_||_nu|jJ|jJ|xjdz c_||jkr|j|j|z |jkr6|xj||z z c_| |||_||_n|xj|j|z z c_| ||j}||_||jz|_n|j||jkr|j|jkrn||jz |jkr6|xj||z z c_| |||_||_nB|xj||jz z c_| |j|}|j|z|_|jt|jz|_||jz }|j|||z|z }|j rT|j|jz |jdzz}|dkr4|xj|j|zz c_|j|j|zd|_|Sr) rr*rcr#rRrrzr%r$rr}r)r&r*rcoffsetbendnewrnums r'r0zBytesCache._fetchsa =E ;)C DI  #3 J "##$dhTZ'F NNa NN:fv|e';;< < > ty#"677DDD 5==EDI--3 J %$*"4"4 H dh  & &$, 6 & & OOq OOeT22DJDJJ:)))8''' OOq OOtz!!8#tx#~'F'F..$,>..!%eT!:!:DJ!&DJJ..$*u2DD..,,udj99C!&DJ!$tz!1DJJ%$//8di''48^dn44..$,>..!%eT!:!:DJ!&DJJ..$/A..,,tx66C!%c!1DJ:DJ/#j&3,"667 9 @8dj(dnq.@ACQww dns22 !Z(<(>(>?  r)c*t|jSrN)r}rRr2s r'__len__zBytesCache.__len__=s4:r))T) rrrrrrrrrrr)rr) r:r?r@rArrBr(r0rrrs@r'rrs  "D!!!!IMGGGGRr)rcFeZdZUdZdZded< ddfd ZddZxZS)AllBytesz!Cache entire contents of the fileallrrNrr+rFetcher | Nonerr bytes | Nonerrct||||@|xjdz c_|xj|jz c_|d|j}||_dS)NrUr)rOr(r$r%rrr)r&rrrrr9s r'r(zAllBytes.__init__Fsl GT222 < OOq OO  & &$) 3 & &<<49--D r)r*r,r-c@|xjdz c_|j||S)NrU)r#rr/s r'r0zAllBytes._fetchTs# !yt$$r))NNNN) rr+rrrr+rrrrr= r:r?r@rArrBr(r0rrs@r'rrAs}++D!%"&!       %%%%%%%%r)rcFeZdZUdZdZded< ddfd Zdfd ZxZS)KnownPartsOfAFilea Cache holding known file parts. Parameters ---------- blocksize: int How far to read ahead in numbers of bytes fetcher: func Function of the form f(start, end) which gets bytes from remote as specified size: int How big this file is data: dict A dictionary mapping explicit `(start, stop)` file-offset tuples with known bytes. strict: bool, default True Whether to fetch reads that go beyond a known byte-range boundary. If `False`, any read that ends outside a known part will be zero padded. Note that zero padding will not be used for reads that begin outside a known byte-range. partsrrNTrrrrrr#dict[tuple[int, int], bytes] | Nonestrictrrr c `t|||||_|rt|}|dg}||dg} |ddD]\} } |d\} } | | kr-| | f|d<| dxx|| | fz cc<C|| | f| || | ftt|| |_ dSi|_ dS)NrrUrq) rOr(rsortedkeyspopr{dictzipr)r&rrrrrr old_offsetsoffsetsrIr*r,start0stop0r9s r'r(zKnownPartsOfAFile.__init__rs4 GT222    --K"1~&Ghh{1~../F*122 ; ; t ' E>>#)4.GBK2JJJ$((E4="9"99JJJJNNE4=111MM$((E4="9"9::::S&1122DIIIDIIIr)r*r+r,rr-c|d}||j}d}|jD]w\\}}}||cxkr|krbn||z }||||z|z }|jr||cxkr|kr2nn/|d||z t |z zz }|xjdz c_|cS|}nx|jtd||fdtj d||fdt d|d ||xj ||z z c_ |xj dz c_ |t||zS) Nrr)rUz&Read is outside the known file parts: z. z%. IO/caching performance may be poor!z!KnownPartsOfAFile cache fetching re)rritemsrr}r#rrwarningswarnrsrtr%r$rOr0) r&r*r,rloc0loc1roffr9s r'r0zKnownPartsOfAFile._fetchs =E <9D"&)//"3"3   LT4$u####t#####dl3te!334{ dd&:&:&:&:d&:&:&:&:&: 7dUlSXX&=>>CNNa'NNJJJ !EE!$* < WudmWWWXX X   3eT] 3 3 3     GGGGGHHH ""dUl2"" 1UWW^^E40000r))NT) rrrrrrrrrrrr r=rrs@r'rrYs,"D!!!!59 <+1+1+1+1+1+1+1+1+1+1r)rcXeZdZdZGddeZddd ZddZddZddZ ddZ dS) UpdatableLRUzg Custom implementation of LRU cache that allows updating keys Used by BackgroudBlockCache c8eZdZUded<ded<ded<ded<dS)UpdatableLRU.CacheInforhitsmissesmaxsizecurrsizeN)r:r?r@rBrCr)r' CacheInfors4    r)r funcCallable[P, T]max_sizerrrctj|_||_||_d|_d|_tj|_ dSr!) collectionsr_cache_func _max_size_hits_misses threadingLock_lock)r&r r s r'r(zUpdatableLRU.__init__sA+6+B+D+D  !  ^%% r)argsP.argskwargsP.kwargsrc&|r$td||j5||jvrC|j||xjdz c_|j|cdddS dddn #1swxYwY|j|i|}|j5||j|<|xjdz c_t|j|j kr|j ddddn #1swxYwY|S)Nz Got unexpected keyword argument rUFlast) TypeErrorrrr move_to_endrrrr}rpopitem)r&rrresults r'__call__zUpdatableLRU.__call__s  PNv{{}}NNOO O Z ) )t{"" ''--- a {4(  ) ) ) ) ) ) ) )" ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) T,V,, Z 0 0 &DK  LLA LL4;$.00 ###///  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0  s%ABB B 'ADD  D r rcV|j5||jvcdddS#1swxYwYdSrN)rr)r&rs r' is_key_cachedzUpdatableLRU.is_key_cachedsx Z ' '4;& ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' 's ""r"c|j5||j|<t|j|jkr|jdddddS#1swxYwYdS)NFr)rrr}rr!)r&r"rs r'add_keyzUpdatableLRU.add_keys Z 0 0 &DK 4;$.00 ###/// 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0sAAAArc|j5||jt|j|j|jcdddS#1swxYwYdS)N)rrrr)rr rr}rrrr2s r'rzUpdatableLRU.cache_infos Z  >>T[))Z| "                  s:AAAN)r )r r r rrr)rrrrrr)rr rr)r"rrr rrrr) r:r?r@rArr r(r#r%r'rrCr)r'rrs J &&&&&&''''0000 r)rcneZdZUdZdZded< d!d"fd Zd#dZd$dZd%dZ d&dZ d'd(fd Z d)d Z xZ S)*BackgroundBlockCachea Cache holding memory as a set of blocks with pre-loading of the next block in the background. Requests are only ever made ``blocksize`` at a time, and are stored in an LRU cache. The least recently accessed block is discarded when more than ``maxblocks`` are stored. If the next block is not in cache, it is loaded in a separate thread in non-blocking way. Parameters ---------- blocksize : int The number of bytes to store in each block. Requests are only ever made for ``blocksize``, so this should balance the overhead of making a request against the granularity of the blocks. fetcher : Callable size : int The total size of the file being cached. maxblocks : int The maximum number of blocks to cache for. The maximum memory use for this cache is then ``blocksize * maxblocks``. backgroundrrrrrrrrrrrc>t|||tj||z |_||_t |j||_td|_ d|_ d|_ tj|_dS)NrU max_workers)rOr(rrr"rrrrr_thread_executor_fetch_future_block_number _fetch_futurerr_fetch_future_lockrs r'r(zBackgroundBlockCache.__init__s GT222y !122 "#/0A9#M#M 2q A A A6:'37"+."2"2r)rc4|jSrrr2s r'rzBackgroundBlockCache.cache_info#rr)rc2|j}|d=|d=|d=|d=|d=|S)Nrr0r1r2r3rrs r'rz!BackgroundBlockCache.__getstate__.s:  ' ( $ % . / / " & ' r)c|j|t|j|d|_t d|_d|_d|_tj |_ dS)NrrUr.) rr|rrrrr0r1r2rrr3rs r'rz!BackgroundBlockCache.__setstate__7sk U####/0A5CU#V#V 2q A A A*.'!"+."2"2r)r*r+rcr-cV|d}||j}||jks||krdS||jz}||jz}d}d}|j5|j|jJ|jr`t d|j |j |jd|_d|_n@t||jcxko|knc}|r|j}|j}d|_d|_dddn #1swxYwY|Gt d|j | |t||dzD]}|||dz} |j5|jR| |j krG|j| s-| |_|j|j| d|_dddn #1swxYwY|||||S)Nrr)z3BlockCache joined background fetch without waiting.z(BlockCache waiting for background fetch.rUasyncr)rrr3r2r1donersrrr'r"rrur"r%r0submitrr) r&r*rcrrfetch_future_block_number fetch_future must_joinrend_block_plus_1s r'r0zBackgroundBlockCache._fetch?s, =E ;)C DI  #3#dn4$.0$(!  $ 2 2!-6BBB%**,,2KK UVVV,44*1133T5T7;D3)-D&&!%*:,,,,+,,,,!!I !2594S1'+'9 ;?7-1*7 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2<  # KKB C C C  $ , ,##%%'@    ""46F6JKK 3 3L  $ $\ 2 2 2 2,a/  $  "*$ 440>>?OPP53C/%)%:%A%A%'7&&"                 1-     s&C DDD!AHH H syncrlog_infor4c>||jkrtd|d|jd||jz}||jz}td|||xj||z z c_|xjdz c_t||}|S)rrrrrz!BlockCache fetching block (%s) %drU) r"rrrsrr%r$rOr0)r&rr@r*rcrr9s r'rz!BackgroundBlockCache._fetch_blocks $, & &999)-999  t~-dn$ 7<PPP ""cEk1"" 1s33r)rrc ||jz}||jz}|xjdz c_||kr||}|||S|||dg}|t |jt |dz||||d|d|Srrrs r'rz BackgroundBlockCache._read_cachesDN* & ! !1 1 1,,-?@@E7*+ +++,>?? KLC JJ,,q02BCC    JJt//0@AA(7(K L L L88C== r)rrr)rr>r)r?)rrr@r4rr-r)r:r?r@rArrBr(rrrr0rrrrs@r'r+r+s2'D&&&&MO 3 3 3 3 3 3 3 5 5 5 53333J J J J X$(!(!(!(!(!(!(!(!r)r+z!dict[str | None, type[BaseCache]]cachesFclstype[BaseCache]clobberrrrc~|j}|s)|tvr td|dt||t|<dS)z'Register' cache implementation. Parameters ---------- clobber: bool, optional If set to True (default is False) - allow to overwrite existing entry. Raises ------ ValueError zCache with name z is already known: N)rrCr)rDrFrs r'register_cacherHsL 8D Wtv~~UDUUvd|UUVVVF4LLLr))F)rDrErFrrr)3 __future__rrrloggingrrYrrrconcurrent.futuresrr itertoolsroperatorrtypingr r r r r rrrFtyping_extensionsrrr getLoggerrsrr-rlist MultiFetcherrrErrrrrrrr+rCrBrHcrCr)r'rTs""""""" ######99999999KKK++++++ #AA A GCLL  8 $ $ C:u$ %c3h(%/0 < < < < < < < < ~K&K&K&K&K& K&K&K&\+%+%+%+%+%Y+%+%+%\+,+,+,+,+,i+,+,+,\F!F!F!F!F!F!F!F!RbbbbbbbbJ%%%%%y%%%0b1b1b1b1b1 b1b1b1J9999971a4=999xK!K!K!K!K!9K!K!K!` )- (     AN1  r)