DUfsddlmZddlZddlZddlmZddlmZmZddl m Z m Z m Z ddl Z ddlZddlZddlZddlmZddlmZdd lmZdd lmZdd lmZmZdd lmZdd l m!Z!m"Z"gdZ#ee$Z%dZ&gdZ'dMdZ(GddeZ) dNdOd%Z*dPd)Z+dQd.Z,dRd2Z-dSd3Z. dTdUd9Z/ dVdWd>Z0Gd?d@eZ1 dXdYdEZ2 dXdZdIZ3 dVd[dLZ4dS)\) annotationsN) bisect_right) OrderedDict defaultdict)AnyIteratorLiteral) get_logger) MapFunctor)__format_version_mcool__Cooler) ContactBinnercreate)lock)GenomeSegmentationparse_cooler_uri) merge_coolerscoarsen_coolerzoomify_cooler) iiii'iaiPiii i@Bi%&i@KLiindexeslist[np.ndarray | h5py.Dataset]bufsizeintreturntuple[np.ndarray, np.ndarray]ctj|dj}tt |D] }|||z }d}|d}dg}dg}d} t |t ||z||dz } | |kr| dz } || ||| || |krn | }|| }ytj|}tj|}tj |} | |k } | dkr0tj | d|d| dd ||fS) a Given ``k`` bin1_offset indexes, determine how to partition the data from ``k`` corresponding pixel tables for a k-way merge. The paritition is a subsequence of bin1 IDs, defining the bounds of chunks of data that will be loaded into memory from each table in a single "epoch" of merging data. The bounds are calculated such that no single epoch will load more than ``bufsize`` records into memory. However, the ``bufsize`` condition is not guaranteed and a warning will be raised if it cannot be satisfied for one or more epochs (see Notes). Parameters ---------- indexes : sequence of 1D array-like of equal length Offset arrays that map bin1 IDs to their offset locations in a corresponding pixel table. bufsize : int Maximum number of pixel records loaded into memory in a single merge epoch. Returns ------- bin1_partition : 1D array Bin1 IDs defining where to partition all the tables for merging. cum_nrecords : 1D array Cumulative number of records (from all pixel tables combined) that will be processed at each epoch. Notes ----- The one exception to the post-condition is when a single bin1 increment in a table contains more than ``bufsize`` records. rT)lor z/ merge epochs will require buffering more than z pixel records, with as many as g.)npzerosshaperangelenrminappendarraydiffsumwarningswarnmax) rrcombined_indexicombined_start combined_nnzbin1_partition cum_nrecordsr!hinrecords_per_epochn_overs J/var/www/html/software/conda/lib/python3.11/site-packages/cooler/reduce.pymerge_breakpointsr;.sVXgaj.//N 3w<< %%'!*$N!"%LSN3L B,   (, 7 7      88 !GBb!!!N2./// "  - -  '+),,Xn--N8L))L.. 7* / / 1 1F zz  L Lg L L.@.D.D.F.F K L L L   < ''c(eZdZdZ ddd ZddZdS) CoolerMergerz, Implementation of cooler merging. Ncoolers list[Cooler]mergebufrcolumnslist[str] | Noneaggdict[str, Any] | Nonecft||_||_|dgn||_d|jD|_||j||dj}|td|Ddkrtd|dj }tdt|D]4}tj ||j |kstd5dS|d gddd}tdt|D]s}|| gddd} t| t|kstj | |kstd tdS) Ncountci|]}|dSr-.0cols r: z)CoolerMerger.__init__..s7773C777r<rch|] }|j SrJ)binsizerLcs r: z(CoolerMerger.__init__..s///!AI///r<r z%Coolers must have the same resolutionz&Coolers must have the same chromosomeschromstartendz$Coolers must have same bin structure)listr?rArBrDupdaterPr( ValueError chromsizesr'r$allbins) selfr?rArBrDrPr[r2r]bins2s r:__init__zCoolerMerger.__init__sG}}   $+Oyy 77$,777 ? HOOC !*$  //w///00144 !HIII .J1c'll++ O Ovgaj3zABBO$%MNNNO O O1:??$$%>%>%>?BD1c'll++ M M ))*C*C*CDQQQGJJ#d))++BF5D=4I4I+$%KLLL, M Mr<rIterator[dict[str, np.ndarray]]c #~ Kd|jD}t||j\}}tj|}t dt|t d|t d|d|jD}t d|dgt|jz}|ddD]NJ fd |D}tj d t|j||Ddd }| d dgd  |j} d| DVt d||}dS)NcDg|]}|ddS)rindexes/bin1_offset)openrQs r: z)CoolerMerger.__iter__..s(LLL!166#;;45LLLr<zn_merge_epochs: zbin1_partition: znrecords_per_merge_epoch: cPg|]#}t|$SrJ)r(pixelsrQs r:rgz)CoolerMerger.__iter__..s&666AAHHJJ666r<znnzs: rr c g|] }| SrJrJ)rLindexbin1_ids r:rgz)CoolerMerger.__iter__..s999U7^999r<c`g|]+\}}}||z dk|||,S)r)ri)rLrRrVstops r:rgz)CoolerMerger.__iter__..sH&5$u ))HHJJuTz*)))r<T)axis ignore_indexrlbin2_idsortc$i|] \}}||jSrJvaluesrLkvs r:rNz)CoolerMerger.__iter__..s 66641a1ah666r<zrecords consumed: )r?r;rAr$r,loggerinfor(debugpdconcatzipgroupby aggregaterD reset_indexitems) r^rr5r6r8nnzsstartsstopscombineddfrls @r:__iter__zCoolerMerger.__iter__sLLt|LLL(9$-'P'P$ W\22 @s+='>'>@@AAA 888999 F2DFFGGG66666 OTOO$$$s4<(((%abb)  G9999999Ey*-dlFE*J*J !H  )Y!7d CC48$$  76288::666 6 6 6 KK4U44 5 5 5FF1  r<)NN)r?r@rArrBrCrDrErra)__name__ __module__ __qualname____doc__r`rrJr<r:r>r>sX%)%) MMMMM<((((((r<r> output_uristr input_uris list[str]rArBrCdtypesrErDNonec @tdd|d|D}d|D}t |rd} n!t |sd} nt d|d g}tt} |D]`} | j } |D]B} | | vrt d | d | j d | |  | | Ca|i}|D]} | |vrtj| | || <|d gddd}|d jdd}t#||||}t%|||f|||| d|dS)a Merge multiple coolers with identical axes. The merged cooler is stored at ``output_uri``. .. versionadded:: 0.8.0 Parameters ---------- output_uri : str Output cooler file path or URI. input_uris : list of str List of input file path or URIs of coolers to combine. mergebuf : int Maximum number of pixels processed at a time. columns : list of str, optional Specify which pixel value columns to include in the aggregation. Default is to use all available value columns. dtypes : dict, optional Specific dtypes to use for value columns. Default is to propagate the current dtypes of the value columns. agg : dict, optional Functions to use for aggregating each value column. Pass the same kind of dict accepted by ``pandas.DataFrame.groupby.agg``. Default is to apply 'sum' to every value column. kwargs Passed to ``cooler.create``. Notes ----- The default output file mode is 'w'. If appending output to an existing file, pass `mode='a'`. See also -------- cooler.coarsen_cooler cooler.zoomify_cooler z Merging: {} c,g|]}t|SrJr)rLpaths r:rgz!merge_coolers..s 0 0 0TF4LL 0 0 0r<c$g|] }|jdkS)symmetric-upper) storage_mode)rLclrs r:rgz!merge_coolers.. s!EEEs#44EEEr<TFz1Cannot merge symmetric and non-symmetric coolers.NrGPixel value column '' not found in input ''.rrTzgenome-assembly)rArBrD)rBrassemblysymmetric_upper)rzr{formatjoinr\anyrZrrXrirfilenamer*r$ result_typer]getr>r)rrrArBrrDkwargsclrsis_symmr dtype_mapr pixel_dtypesrMr]riterators r:rrsb KK%%dii &;&;<<=== 0 0Z 0 0 0DEEEEEG 7||N \\NLMMM)D!!I 9 9zz||*  9 9C,&& /3//!l/// #%%l3&78888 9~:: f  .)C.9F3K 7<<>>333 4QQQ 7DAw| 1488HD8W#NNNH   '        r<edges np.ndarraymaxlenc tj|}t|dkr |ddksJtjdtjtj|f}fdt dttj|dz D}| |dtj tj ||}||S)zGiven an integer interval partition ``edges`` from 0..nnz, prune the edges to make the new subintervals roughly ``maxlen`` in length. rcg|]}|zSrJrJ)rLr2rs r:rgz+_greedy_prune_partition..?s L L L1FQJ L L Lr<r ) r$asarrayr(r_cumsumr,r'rceilr*unique searchsorted)rrcumlencutsidxs ` r:_greedy_prune_partitionr8s Ju  E u::??uQx1}}}} U1bi/// 0F L L L LaRWVBZ&5H-I-I)J)J K K L L LDKKr  )BOFD11 2 2C :r<r[ pd.Series base_binsize bins_per_tilec||z}t|}tj||z }ttjt j|}|S)aH Number of zoom levels for a quad-tree tiling of a genomic heatmap. At the base resolution, we need N tiles, where N is the smallest power of 2 such that the tiles fully cover the 1D data extent. From that starting point, determine the number of zoom levels required to "coarsen" the map up to 1 tile. )r-mathrrr$log2)r[rrtile_length_bptotal_bpn_tiles n_zoom_levelss r:get_quadtree_depthrEsX#\1N:Hi>122G "''"2"23344M r<rVmul Iterator[int]c#`Kt|t|}}|V ||z}|V )z Generate a geometric progression of integers. Beginning with integer ``start``, yield an unbounded geometric progression with integer ratio ``mul``. rrVrs r:geomprogr`sBUSXX3E KKK   r<c#RKt|}|V dD] }||zV |dz})z Generate a nice progression of integers. Beginning with integer ``start``, yield a sequence of "nicely" spaced integers: an unbounded geometric progression with ratio 10, interspersed with steps of ratios 2 and 5. T)r rrrs r:niceprogrosQ JJE KKK  C#+      r<nicernstyleLiteral['binary', 'nice'] list[int]c||krgS|dkrt|d}n)|dkrt|}ntd|dt|g} t|}||krn||,|S)a Return a sequence of integers with a "preferred" stepping pattern. Parameters ---------- start : int Starting value in the progression. stop : int Upper bound of progression, inclusive. Values will not exceed this. style : {'nice', 'binary'} Style of progression. 'nice' gives geometric steps of 10 with 2 and 5 in between. 'binary' gives geometric steps of 2. Returns ------ list of int Examples -------- For certain values of `start` (n * 10^i), nice stepping produces familiar "preferred" sequences [1]_: Note denominations in Dollars (1-2-5) >>> preferred_sequence(1, 100, 'nice') [1, 2, 5, 10, 20, 50, 100] Coin denominations in Cents >>> preferred_sequence(5, 100, 'nice') [5, 10, 25, 50, 100] .. [1] https://en.wikipedia.org/wiki/Preferred_number#1-2-5_series binaryrrz1Expected style value of 'binary' or 'nice'; got 'r)rrrZnextr*)rVrnrgenseqns r:preferred_sequencersR t||  ua   &uooPuPPPQQQ 99+C II t88  1  Jr< resolutionsbaseslist[int] | None)tuple[np.ndarray, np.ndarray, np.ndarray]c|t|h}nt|}tjt ||}tjt|t }tjt|t }tt|dddD]>\}}|dz }|dkr.|||zdkr|||<|||z||<n |dz}|dk.?t|D]1\}}|dkr&|||vrtd||d|d2|||fS) a" From a set of target resolutions and one or more base resolutions deduce the most efficient sequence of integer multiple aggregations to satisfy all targets starting from the base resolution(s). Parameters ---------- resolutions: sequence of int The target resolutions bases: sequence of int, optional The base resolutions for which data already exists. If not provided, the smallest resolution is assumed to be the base. Returns ------- resn: 1D array Resolutions, sorted in ascending order. pred: 1D array Index of the predecessor resolution in `resn`. A value of -1 implies that the resolution is a base resolution. mult: 1D array Multiplier to go from predecessor to target resolution. N)dtyper r rz Resolution z. cannot be derived from the base resolutions: r#) r)setr$r+sorteduniononesr(rrX enumeraterZ)rrresnpredmultr2targetps r:get_multiplier_sequencers8 }[!!"E  8F5;;{3344 5 5D GCIIS ) ) ) )D GCIIS ) ) ) )D)D//**44R40 6 E1ffQ1$$Q DG+QQ 1ff$1 77tAwe++2d1g22).222  t r<cNeZdZdZefddZeddZddZddZ ddZ dS) CoolerCoarsenerz/ Implementation of cooler coarsening. source_urirfactorr chunksizerBrrDrE batchsizemapr cB||_||_||_t|tr|dksJ||_t ||_ddg|_t||_ |j|j z|_ d|j D|_ ||j |t|}|j} |j|_|d|_|d|_|jd|_n|j|z|_|gddd} || | ||_t1| |j|_g} |jjD]E\} } |j| }|j| dz}| |j|||F| |jdt=| |j|_dS) Nr rlrqci|]}|dSrIrJrKs r:rNz,CoolerCoarsener.__init__..s===3C===r<zindexes/chrom_offsetrerTr ) _maprr isinstancerrr index_columnsrX value_columnsrBrDrYrr[rP old_binsize _load_dsetold_chrom_offsetold_bin1_offset new_binsizer] coarsen_binsnew_binsrgsidmaprextendr*rr)r^rrrrBrDrrrr[old_binsr_chromr2c0c1s r:r`zCoolerCoarsener.__init__s $"&#&&56A:::: Y'3!']])D,>> ==$*<=== ? HOOC Z  ^ ; #/E F F"~~.CDD   ##D  #/&8D 88::7778;))(JGG $Z?? ,,.. = =IFA&q)B&q1u-B LL-bFl; < < < < T)"-...,UDNCC r<r pd.DataFramer[rrcfd}|ddgd|dS)Nc |ddgjdd}|djdz dj}t|t|krtj||jf}||d<|S)NrUrVrWr )copyilocrvr(r$rname)groupoutrWr[rs r:_eachz+CoolerCoarsener.coarsen_bins.._each9s'*+00227&AC,#FQJ$8&$89@C3xx#c((""eCEJ!778CJJr<rUTobservedrT)drop)rapplyr)rr[rrs `` r:rzCoolerCoarsener.coarsen_bins5sf        WWtW , ,-F-F-F H U5\\ [d[ # #  r<spantuple[int, int]c|\}}t|j}|dd|j}|||}t|d||jj}|jj}|jj } |jj } |dj } |dj } |dj } |dj }|Q| | | z}| | |z}tj | |d d z |d <tj | |d d z |d <nztj| |z t }tj||z t }|| |z|d <|| |z|d <||jd|jS)NTF)r convert_enum chrom1chrom2start1start2right)sider rlrqrr)rrrirBrzr{rrPchrom_binoffset chrom_abspos start_absposrvr$rfloorastyperrrrrDr)r^rr!r7rtablechunkrPr"r#r$ chrom_id1 chrom_id2rr abs_start1 abs_start2rel_bin1rel_bin2s r: _aggregatezCoolerCoarsener._aggregateHsBT_%% 5 99$,Gbe  rLLBLL!!!'/'1w+ w+ (O* (O* x'x' ?%i069J%i069J jwGGG!K )  jwGGG!K )  x 01188==Hx 01188==H.y9HDE) .y9HDE)  MM$,4M 8 8 Ytx []] r<c ||}n/#t$r"}tt||d}~wwxYw|SN)r/ MemoryError RuntimeErrorr)r^rr(es r:rzCoolerCoarsener.aggregatepsT .OOD))EE . . .s1vv&&A - . s A?Arac#K|j}tt|jdd|jdd}t dt ||D]} |dkrt j||j ||||z}|dkrt j n#|dkrt j wwxYw|D]"}d| DV#dS)Nr r rc$i|] \}}||jSrJrurws r:rNz,CoolerCoarsener.__iter__..s :::tq!q!(:::r<) rrXrrr'r(racquirerrreleaser)r^rspansr2resultsrs r:rzCoolerCoarsener.__iter__wsN SCRC$*QRR.99::q#e**i00 ; ;A #q==LNNN))DNE!a)m:K4LMMq==LNNNq==LNNNN! ; ;::rxxzz::::::: ; ; ;s "?B;;CN)rrrrrrrBrrDrErrrr )rr r[rrrrr )rrrr r) rrrrrr` staticmethodrr/rrrJr<r:rrs6D6D6D6D6Dp   \ $& & & & P ; ; ; ; ; ;r<rbase_urirrnprocc t|} t|}|dg}|i}| j} |D]=} | | vrt d| d| jd|| | | > |dkr/tj|} |dtt|||||||dkr| j nt} | j }|d d t||| f|| jd kd ||dkr| dSdS#|dkr| wwxYw) aP Coarsen a cooler to a lower resolution by an integer factor *k*. This is done by pooling *k*-by-*k* neighborhoods of pixels and aggregating. Each chromosomal block is coarsened individually. Result is a coarsened cooler stored at ``output_uri``. .. versionadded:: 0.8.0 Parameters ---------- base_uri : str Input cooler file path or URI. output_uri : str Input cooler file path or URI. factor : int Coarsening factor. chunksize : int Number of pixels processed at a time per worker. nproc : int, optional Number of workers for batch processing of pixels. Default is 1, i.e. no process pool. columns : list of str, optional Specify which pixel value columns to include in the aggregation. Default is to use all available value columns. dtypes : dict, optional Specific dtypes to use for value columns. Default is to propagate the current dtypes of the value columns. agg : dict, optional Functions to use for aggregating each value column. Pass the same kind of dict accepted by ``pandas.DataFrame.groupby.agg``. Default is to apply 'sum' to every value column. kwargs Passed to ``cooler.create``. See also -------- cooler.zoomify_cooler cooler.merge_coolers NrGrrrr r)rBrDrrr*Tr)rr)rrrirrZr setdefaultmpPoolrrrrrrclose)r<rrrr=rBrrDrr input_dtypesrMpoolrrs r:rrsl   C [[F) ~::<<&L66 l " "VsVVclVVV    c<#4 5 5 5 5 19975>>D   fd + + +"   !AII3   $(D)))    ,0AA        199 JJLLLLL 95199 JJLLLL s BD55E base_urisstr | list[str]outfilec t|tr|g}i} i} t} |D]} t| \} }t | |}|jdn|j}| |f| |<|dddd | |<| |t|| \}}}t|}td|d|dg}| D]R}td t|z| |\} }tj| d 5}tj|d 5}d |}||d z||d z||dz||dzddgt%|D]%}||d|z||d|z&||dz||dz||j||jdddn #1swxYwYdddn #1swxYwYTt+|D]q}||dkr|||}|||z}td|d|dt-|d|z|d|z|||f||||dd|rtj|d5}|jdt.dddddS#1swxYwYdS)a Generate multiple cooler resolutions by recursive coarsening. Result is a "zoomified" or "multires" cool file stored at ``outfile`` using the MCOOL v2 layout, where coolers are stored under a hierarchy of the form ``resolutions/`` for each resolution ``r``. .. versionadded:: 0.8.0 Parameters ---------- base_uris : str or sequence of str One or more cooler URIs to use as "base resolutions" for aggregation. outfile : str Output multires cooler (mcool) file path. resolutions : list of int A list of target resolutions to generate. chunksize : int Number of pixels processed at a time per worker. nproc : int, optional Number of workers for batch processing of pixels. Default is 1, i.e. no process pool. columns : list of str, optional Specify which pixel value columns to include in the aggregation. Default is to use only the column named 'count' if it exists. dtypes : dict, optional Specific dtypes to use for value columns. Default is to propagate the current dtypes of the value columns. agg : dict, optional Functions to use for aggregating each value column. Pass the same kind of dict accepted by ``pandas.DataFrame.groupby.agg``. Default is to apply 'sum' to every value column. kwargs Passed to ``cooler.create``. See also -------- cooler.coarsen_cooler cooler.merge_coolers Nr rUTrz$Copying base matrices and producing z new zoom levels.rGz Bin size: rdwz /resolutions/z/chromsz/binsrlrqz/pixels/z/indexesr zAggregating from z to r#z::resolutions/r+)r=rBrrDmodez HDF5::MCOOL)rzformat-version)rrrrrrPr]rsizer0addrr(rzr{h5pyFiler rXattrsrYr'rr )rErGrrr=rBrrDr parsed_urisn_bins_longest_chrombase_resolutions input_uriinfileingrouprrrrrn_zoomssrcdestprefixrMr2 prev_binsizerPfws r:rrsj)S!! K Kuu + + *955VW%%K/qqS[ %+W$5 L! HHJJqqqM WWtW , , TVV SUU \* \****/{ ? ? ?!9=tG}}=  ....---- HHWz)4*1D E E E L  % %c'l&8 9 9 9 : : : : : : : : : : : : : : : : : : : : : : : : : : : : : :"7^^   7b== DG} a( D DD'DDDEEE 5|55 5 0w00 0 G     7D ! ! R $8P Q Q                     sI+I%CI I%I I%I I%%I) ,I) #L88L<?L<rTtuple[int, dict[str, int]]c t|\}}t||}t|j|jt }d} t dtj |jt d|jt d|t dd|zt d|d|dzd zt} t|} |j} t d t| zd zt| ztj |d 5} tj |d 5}| ||t| | | | <dddn #1swxYwYdddn #1swxYwYt|dz ddD]}| | z} t|dz}t|} t dt| zd zt| zt!|dzt|z|dzt| z| |||| | | <tj |d5}|jd|i||jd<|j| dddn #1swxYwY|| fS)zL Quad-tree tiling using legacy MCOOL layout (::0, ::1, ::2, etc.). rztotal_length (bp): z binsize: z n_zooms: zquad tile cover: zCopying base matrix to level z and producing z new zoom levels zcounting down to 0...z Zoom level: z bin size: rdrINr r zAggregating at zoom level: z::)rr=rrJzmax-zoomz max-zooms)rrrr[rPHIGLASS_TILE_DIMrzr{r$r-rrrNrOr r'rrPrY)rTrGr=rrrUrVrrWr zoom_levels zoomLevelrPrXrYr2 prevLevelr\s r:legacy_zoomifyrcrs'y11OFG  ! !C >NOOG F KK>bfS^&<&<>>??? KK)CK))*** KK%G%%&&& KK2AL22333 KK'  ? ?W ? ? ? @ ! " --KG IkG KKY/-?#g,,NOOO 63  )3 7C )$( $I///!( I))))))))))))))))))))))))))))))7Q;B ' '))6AJJ FF  ))nn  'll      dNS^^ + dNS^^ +      ") I 7D ! !%R W-... '  $$$ %%%%%%%%%%%%%%% K sI4G *G4 GG GG GGG1AK>>LL)rrrrrr)NNN)rrrrrArrBrCrrErDrErr)rrrrrr)r[rrrrrrr)rVrrrrr)rVrrr)r)rVrrnrrrrrr1)rrrrrr)r NNN)r<rrrrrrrr=rrBrCrrErDrErr)rErFrGrrrrrr=rrBrCrrErDrErr) rTrrGrr=rrrrr])5 __future__rrr.bisectr collectionsrrtypingrrr rN multiprocessr@numpyr$pandasr}_loggingr _typingr _versionr apirrrparallelrutilrr__all__rrzr_ ZOOMS_4DNr;r>rrrrrrrrrrrcrJr<r:rss"""""" 00000000))))))))))  ......))))))))66666666 ? ? ? H      "U(U(U(U(pLLLLL=LLLf!%$(!% `````F    6    ((.:::::~#77777tM;M;M;M;M;mM;M;M;j $$(!%hhhhh` $$(!%} } } } } BDH@ @ @ @ @ @ @ r<