o Nrfs@sddlmZddlZddlZddlmZddlmZmZddl m Z m Z m Z ddl Z ddlZddlZddlZddlmZddlmZdd lmZdd lmZdd lmZmZdd lmZdd l m!Z!m"Z"gdZ#ee$Z%dZ&gdZ'dXddZ(GdddeZ)   dYdZd&d'Z*d[d+d,Z+d\d1d2Z,d]d6d7Z-d^d8d9Z. :d_d`d?d@Z/ dadbdEdFZ0GdGdHdHeZ1    dcdddMdNZ2    dcdedRdSZ3 dadfdVdWZ4dS)g) annotationsN) bisect_right) OrderedDict defaultdict)AnyIteratorLiteral) get_logger) MapFunctor)__format_version_mcool__Cooler) ContactBinnercreate)lock)GenomeSegmentationparse_cooler_uri) merge_coolerscoarsen_coolerzoomify_cooler) iiii'iaiPiii i@Bi%&i@KLiindexeslist[np.ndarray | h5py.Dataset]bufsizeintreturntuple[np.ndarray, np.ndarray]c Cst|dj}tt|D]}|||7}qd}|d}dg}dg}d} t|t||||dd} | |kr<| d7} || ||| || |krOn| }|| }q&t|}t|}t |} | |k } | dkrt | d|d| dd ||fS) a Given ``k`` bin1_offset indexes, determine how to partition the data from ``k`` corresponding pixel tables for a k-way merge. The paritition is a subsequence of bin1 IDs, defining the bounds of chunks of data that will be loaded into memory from each table in a single "epoch" of merging data. The bounds are calculated such that no single epoch will load more than ``bufsize`` records into memory. However, the ``bufsize`` condition is not guaranteed and a warning will be raised if it cannot be satisfied for one or more epochs (see Notes). Parameters ---------- indexes : sequence of 1D array-like of equal length Offset arrays that map bin1 IDs to their offset locations in a corresponding pixel table. bufsize : int Maximum number of pixel records loaded into memory in a single merge epoch. Returns ------- bin1_partition : 1D array Bin1 IDs defining where to partition all the tables for merging. cum_nrecords : 1D array Cumulative number of records (from all pixel tables combined) that will be processed at each epoch. Notes ----- The one exception to the post-condition is when a single bin1 increment in a table contains more than ``bufsize`` records. rT)lor z/ merge epochs will require buffering more than z pixel records, with as many as g.)npZzerosshaperangelenrminappendarraydiffsumwarningswarnmax) rrZcombined_indexiZcombined_startZ combined_nnzbin1_partition cum_nrecordsrhinrecords_per_epochZn_overr3V/var/www/html/software/conda/envs/catlas/lib/python3.10/site-packages/cooler/reduce.pymerge_breakpoints.sH+        r5c@s*eZdZdZ  ddd d ZdddZdS) CoolerMergerz, Implementation of cooler merging. Ncoolers list[Cooler]mergebufrcolumnslist[str] | Noneaggdict[str, Any] | Nonec Cs.t||_||_|durdgn||_dd|jD|_|dur%|j||dj}|dur\tdd|Ddkr=td|dj }t dt|D]}t ||j |ksYtd qIdS|d gd dd}t dt|D]#}|| gd dd} t| t|kst | |kstd qqdS) NcountcSi|]}|dqSr*r3.0colr3r3r4 z)CoolerMerger.__init__..rcSsh|]}|jqSr3)binsizerBcr3r3r4 rEz(CoolerMerger.__init__..r z%Coolers must have the same resolutionz&Coolers must have the same chromosomeschromstartendz$Coolers must have same bin structure)listr7r9r:r<updaterFr% ValueError chromsizesr$r"allbins) selfr7r9r:r<rFrQr.rSZbins2r3r3r4__init__s.    zCoolerMerger.__init__rIterator[dict[str, np.ndarray]]c #s"dd|jD}t||j\}}t|}tdt|td|td|dd|jD}td|dgt|j}|d dD]@fd d|D}t j d dt |j||Ddd d }|j ddgd d |j} dd| DVtd||}qNdS)NcSsg|] }|ddqS)rindexes/bin1_offset)openrGr3r3r4 sz)CoolerMerger.__iter__..zn_merge_epochs: zbin1_partition: znrecords_per_merge_epoch: cSsg|]}t|qSr3)r%pixelsrGr3r3r4rZznnzs: rr csg|]}|qSr3r3)rBindexbin1_idr3r4rZcSs.g|]\}}}||dkr|||qS)r)r[)rBrHrLstopr3r3r4rZs  T)ZaxisZ ignore_indexr_bin2_idsortcSi|]\}}||jqSr3valuesrBkvr3r3r4rDr\z)CoolerMerger.__iter__..zrecords consumed: )r7r5r9r"r)loggerinfor%debugpdconcatzipgroupby aggregater< reset_indexitems) rTrr/r0r2ZnnzsZstartsstopscombineddfr3r^r4__iter__s6   zCoolerMerger.__iter__)NN)r7r8r9rr:r;r<r=rrV)__name__ __module__ __qualname____doc__rUrxr3r3r3r4r6s   r6 output_uristr input_uris list[str]r9r:r;dtypesr=r<NonecKsLtdd|dd|D}dd|D}t|r d} n t|s'd} ntd|d ur2d g}tt} |D]%} | j } |D]} | | vrStd | d | j d | |  | | qAq8|d urdi}|D]} | |vrut j| | || <qf|dgdd d }|djdd }t||||d}t|||f|||| d|d S)a Merge multiple coolers with identical axes. The merged cooler is stored at ``output_uri``. .. versionadded:: 0.8.0 Parameters ---------- output_uri : str Output cooler file path or URI. input_uris : list of str List of input file path or URIs of coolers to combine. mergebuf : int Maximum number of pixels processed at a time. columns : list of str, optional Specify which pixel value columns to include in the aggregation. Default is to use all available value columns. dtypes : dict, optional Specific dtypes to use for value columns. Default is to propagate the current dtypes of the value columns. agg : dict, optional Functions to use for aggregating each value column. Pass the same kind of dict accepted by ``pandas.DataFrame.groupby.agg``. Default is to apply 'sum' to every value column. kwargs Passed to ``cooler.create``. Notes ----- The default output file mode is 'w'. If appending output to an existing file, pass `mode='a'`. See also -------- cooler.coarsen_cooler cooler.zoomify_cooler z Merging: {} cSsg|]}t|qSr3r )rBpathr3r3r4rZr`z!merge_coolers..cSsg|]}|jdkqS)symmetric-upper) storage_mode)rBclrr3r3r4rZ sTFz1Cannot merge symmetric and non-symmetric coolers.Nr>Pixel value column '' not found in input ''.rrJzgenome-assembly)r9r:r<)r:rassemblysymmetric_upper)rkrlformatjoinrRanyrPrrNr[rfilenamer'r"Z result_typerSgetr6r)r~rr9r:rr<kwargsZclrsZis_symmrZ dtype_maprZ pixel_dtypesrCrSriteratorr3r3r4rsV1   redges np.ndarraymaxlencst|}t|dkr|ddksJtjdtt|f}fddtdtt|dD}| |dt t ||}||S)zGiven an integer interval partition ``edges`` from 0..nnz, prune the edges to make the new subintervals roughly ``maxlen`` in length. rcsg|]}|qSr3r3)rBr.rr3r4rZ?r`z+_greedy_prune_partition..r) r"Zasarrayr%r_Zcumsumr)r$rceilr'unique searchsorted)rrZcumlencutsidxr3rr4_greedy_prune_partition8s *rrQ pd.Series base_binsize bins_per_tilecCs6||}t|}t||}ttt|}|S)aH Number of zoom levels for a quad-tree tiling of a genomic heatmap. At the base resolution, we need N tiles, where N is the smallest power of 2 such that the tiles fully cover the 1D data extent. From that starting point, determine the number of zoom levels required to "coarsen" the map up to 1 tile. )r*mathrrr"log2)rQrrZtile_length_bpZtotal_bpZn_tilesZ n_zoom_levelsr3r3r4get_quadtree_depthEs rrLmul Iterator[int]ccs,t|t|}}|V ||9}|Vq)z Generate a geometric progression of integers. Beginning with integer ``start``, yield an unbounded geometric progression with integer ratio ``mul``. rrLrr3r3r4geomprog`srccs0t|}|V dD]}||Vq |d9}q )z Generate a nice progression of integers. Beginning with integer ``start``, yield a sequence of "nicely" spaced integers: an unbounded geometric progression with ratio 10, interspersed with steps of ratios 2 and 5. T)r rrrr3r3r4niceprogos  rnicerastyleLiteral['binary', 'nice'] list[int]cCsp||krgS|dkrt|d}n|dkrt|}ntd|dt|g} t|}||kr2 |S||q')a Return a sequence of integers with a "preferred" stepping pattern. Parameters ---------- start : int Starting value in the progression. stop : int Upper bound of progression, inclusive. Values will not exceed this. style : {'nice', 'binary'} Style of progression. 'nice' gives geometric steps of 10 with 2 and 5 in between. 'binary' gives geometric steps of 2. Returns ------ list of int Examples -------- For certain values of `start` (n * 10^i), nice stepping produces familiar "preferred" sequences [1]_: Note denominations in Dollars (1-2-5) >>> preferred_sequence(1, 100, 'nice') [1, 2, 5, 10, 20, 50, 100] Coin denominations in Cents >>> preferred_sequence(5, 100, 'nice') [5, 10, 25, 50, 100] .. [1] https://en.wikipedia.org/wiki/Preferred_number#1-2-5_series binaryrrz1Expected style value of 'binary' or 'nice'; got 'r)rrrPnextr')rLrargenseqnr3r3r4preferred_sequences)    r resolutionsbaseslist[int] | None)tuple[np.ndarray, np.ndarray, np.ndarray]cCs |dur t|h}nt|}tt||}tjt|td }tjt|td }t t |dddD])\}}|d}|dkr`|||dkrX|||<|||||<n|d8}|dksCq7t |D]\}}|dkr|||vrt d||d|dqe|||fS) a" From a set of target resolutions and one or more base resolutions deduce the most efficient sequence of integer multiple aggregations to satisfy all targets starting from the base resolution(s). Parameters ---------- resolutions: sequence of int The target resolutions bases: sequence of int, optional The base resolutions for which data already exists. If not provided, the smallest resolution is assumed to be the base. Returns ------- resn: 1D array Resolutions, sorted in ascending order. pred: 1D array Index of the predecessor resolution in `resn`. A value of -1 implies that the resolution is a base resolution. mult: 1D array Multiplier to go from predecessor to target resolution. N)Zdtyperr rz Resolution z. cannot be derived from the base resolutions: r!) r&setr"r(sortedunionZonesr%rrN enumeraterP)rrresnpredmultr.targetpr3r3r4get_multiplier_sequences2   rc@sJeZdZdZefd!ddZed"ddZd#ddZd#ddZ d$ddZ d S)%CoolerCoarsenerz/ Implementation of cooler coarsening. source_urirfactorr chunksizer:rr<r= batchsizemapr cCsl||_||_||_t|tr|dksJ||_t||_ddg|_t||_ |j|j |_ dd|j D|_ |dur@|j |t |}|j} |j|_|d|_|d|_|jdur`d|_n|j||_|gddd} || | ||_t| |j|_g} |jjD]\} } |j| }|j| d}| |j|||q| |jd t| |j|_dS) Nr r_rbcSr?r@r3rAr3r3r4rDrEz,CoolerCoarsener.__init__..zindexes/chrom_offsetrXrJr) _maprr isinstancerrr index_columnsrNZ value_columnsr:r<rOrrQrFZ old_binsizeZ _load_dsetZold_chrom_offsetZold_bin1_offsetZ new_binsizerS coarsen_binsnew_binsrgsZidmaprtextendr'rr)rTrrrr:r<rrrrQold_binsrZ_chromr.Zc0c1r3r3r4rUs<          zCoolerCoarsener.__init__r pd.DataFramerQrrcs2fdd}|jdddgd|jddS)Ncsf|ddgjdd}|djddj}t|t|kr-tj||jf}||d<|S)NrKrLrMr )copyZilocrgr%r"rname)groupoutrMrQrr3r4_each9s z+CoolerCoarsener.coarsen_bins.._eachrKTZobservedrJ)Zdrop)rqapplyrs)rrQrrr3rr4r5s zCoolerCoarsener.coarsen_binsspantuple[int, int]cCsD|\}}t|j}|jddd|j}|||}t|d||jj}|jj}|jj } |jj } |dj } |dj } |dj } |dj }|durp| | | }| | |}t j | |d d d |d <t j | |d d d |d <n$t | |t}t ||t}|| ||d <|| ||d <|j|jdd|jS)NTF)rZ convert_enum Zchrom1Zchrom2start1start2right)Zsider r_rbrc)rrr[r:rkrlrrFchrom_binoffset chrom_abspos start_absposrgr"rfloorZastyperrqrrrr<rs)rTrrr1rtablechunkrFrrrZ chrom_id1Z chrom_id2rrZ abs_start1Z abs_start2Zrel_bin1Zrel_bin2r3r3r4 _aggregateHs8        zCoolerCoarsener._aggregatec Cs8z||}W|Sty}ztt||d}~wwN)r MemoryError RuntimeErrorr)rTrrer3r3r4rrps zCoolerCoarsener.aggregaterVc cs|j}tt|jdd|jdd}tdt||D];}z|dkr(t||j ||||}W|dkr>t n |dkrHt ww|D] }dd| DVqKqdS)Nrr rcSrer3rfrhr3r3r4rDr\z,CoolerCoarsener.__iter__..) rrNrprr$r%racquirerrrreleasert)rTrspansr.resultsrwr3r3r4rxws$" zCoolerCoarsener.__iter__N)rrrrrrr:rr<r=rrrr )rrrQrrrrr)rrrrry) rzr{r|r}rrU staticmethodrrrrrxr3r3r3r4rs 8   (rbase_urirrnprocc Kst|} t|}|durdg}|duri}| j} |D]} | | vr.td| d| jd|| | | qzG|dkrGt|} |dt t |||||||dkrU| j nt d} | j }|d d t ||| f|| jd kd |W|dkr}| dSdS|dkr| ww) aP Coarsen a cooler to a lower resolution by an integer factor *k*. This is done by pooling *k*-by-*k* neighborhoods of pixels and aggregating. Each chromosomal block is coarsened individually. Result is a coarsened cooler stored at ``output_uri``. .. versionadded:: 0.8.0 Parameters ---------- base_uri : str Input cooler file path or URI. output_uri : str Input cooler file path or URI. factor : int Coarsening factor. chunksize : int Number of pixels processed at a time per worker. nproc : int, optional Number of workers for batch processing of pixels. Default is 1, i.e. no process pool. columns : list of str, optional Specify which pixel value columns to include in the aggregation. Default is to use all available value columns. dtypes : dict, optional Specific dtypes to use for value columns. Default is to propagate the current dtypes of the value columns. agg : dict, optional Functions to use for aggregating each value column. Pass the same kind of dict accepted by ``pandas.DataFrame.groupby.agg``. Default is to apply 'sum' to every value column. kwargs Passed to ``cooler.create``. See also -------- cooler.zoomify_cooler cooler.merge_coolers Nr>rrrr r)r:r<rrr'Tr)rr)rrr[rrPr setdefaultmpZPoolrrrrrrclose)rr~rrrr:rr<rrZ input_dtypesrCpoolrrr3r3r4rsV6      r base_urisstr | list[str]outfilec Kst|tr|g}i} i} t} |D]5} t| \} }t| |}|jdur%dn|j}| |f| |<|ddjddd | |<| |qt || \}}}t |}t d|d|durcdg}| D]}t d t|| |\} }t| d k}t|d U}d |}||d ||d ||d||dddgt|D]}||d|||d|q||d||d||j||jWdn1swYWdn1swYqet|D]>}||dkrq|||}|||}t d|d|dt|d||d||||f||||dd|qt|d}|jdtdWddS1sOwYdS)a Generate multiple cooler resolutions by recursive coarsening. Result is a "zoomified" or "multires" cool file stored at ``outfile`` using the MCOOL v2 layout, where coolers are stored under a hierarchy of the form ``resolutions/`` for each resolution ``r``. .. versionadded:: 0.8.0 Parameters ---------- base_uris : str or sequence of str One or more cooler URIs to use as "base resolutions" for aggregation. outfile : str Output multires cooler (mcool) file path. resolutions : list of int A list of target resolutions to generate. chunksize : int Number of pixels processed at a time per worker. nproc : int, optional Number of workers for batch processing of pixels. Default is 1, i.e. no process pool. columns : list of str, optional Specify which pixel value columns to include in the aggregation. Default is to use only the column named 'count' if it exists. dtypes : dict, optional Specific dtypes to use for value columns. Default is to propagate the current dtypes of the value columns. agg : dict, optional Functions to use for aggregating each value column. Pass the same kind of dict accepted by ``pandas.DataFrame.groupby.agg``. Default is to apply 'sum' to every value column. kwargs Passed to ``cooler.create``. See also -------- cooler.coarsen_cooler cooler.merge_coolers Nr rKTrz$Copying base matrices and producing z new zoom levels.r>z Bin size: rWwz /resolutions/z/chromsz/binsr_rbz/pixels/z/indexesrzAggregating from z to r!z::resolutions/r+)rr:rr<modez HDF5::MCOOL)rzformat-version)rrrrrrFrSrqsizer-addrr%rkrlh5pyFilerrNattrsrOr$rr )rrrrrr:rr<rZ parsed_urisZn_bins_longest_chromZbase_resolutions input_uriinfileingrouprrrrrn_zoomssrcdestprefixrCr.Z prev_binsizerFfwr3r3r4rs 5                 $rrtuple[int, dict[str, int]]c Cs t|\}}t||}t|j|jt}d} tdt |jtd|jtd|tdd|td|d|dd t } t |} |j} td t | d t | t |d +} t |d }| ||t | | | | <Wdn1swYWdn1swYt|dddD]9}| | 9} t |d}t |} tdt | d t | t|dt ||dt | | |||d| | | <qt |d}|jd|i||jd<|j| Wd|| fS1swY|| fS)zL Quad-tree tiling using legacy MCOOL layout (::0, ::1, ::2, etc.). rztotal_length (bp): z binsize: z n_zooms: zquad tile cover: zCopying base matrix to level z and producing z new zoom levels zcounting down to 0...z Zoom level: z bin size: rWrNr rzAggregating at zoom level: z::)rrrrzmax-zoomz max-zooms)rrrrQrFHIGLASS_TILE_DIMrkrlr"r*rrrrrr$rrrO)rrrrrrrrr rZ zoom_levelsZ zoomLevelrFr r r.Z prevLevelr r3r3r4legacy_zoomifyrst        r)rrrrrr)NNN)r~rrrr9rr:r;rr=r<r=rr)rrrrrr)rQrrrrrrr)rLrrrrr)rLrrr)r)rLrrarrrrrr)rrrrrr)r NNN)rrr~rrrrrrrr:r;rr=r<r=rr)rrrrrrrrrrr:r;rr=r<r=rr) rrrrrrrrrr)5 __future__rrr+bisectr collectionsrrtypingrrrrZ multiprocessrnumpyr"ZpandasrnZ_loggingr Z_typingr _versionr apirrrZparallelrutilrr__all__rzrkrZ ZOOMS_4DNr5r6rrrrrrrrrrrr3r3r3r4sd        XS c   ? : p