o Nrf@s~ddlmZddlZddlmZddlZddlZddlZddl m Z ddl m Z m Z ddl Z ddlZddlZddlZddlmZddlmZddlmZmZmZdd lmZmZdd lm Z m!Z!m"Z"m#Z#m$Z$m%Z%d d l&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3d d l4m5Z5edZ6dxddZ7 dydzdd Z8d{d(d)Z9d|d2d3Z:d}d6d7Z;d~d9d:ZZ=dddBdCZ>ddFdGZ? dddIdJZ@ddNdOZAddPdQZB           ?  ? ? ddRdSZC    T   UddVdWZD ddXdYZEdZFZGd[FZHd\d]ZIeIeGeHd^     ?  _ T   U    ?  dddrdsZJeIeGeHd^     ?  _ T   U    ?  dddvdwZKdS)) annotationsN)datetime)AnyIterable) get_logger)Tabular)__format_version____format_version_scool__ __version__)getput) get_binsizeget_chromsizesget_meta infer_metaparse_cooler_urirlencode) BIN1OFFSET_DTYPE BIN_DTYPE CHROM_DTYPE CHROMID_DTYPECHROMOFFSET_DTYPECHROMSIZE_DTYPE COORD_DTYPE COUNT_DTYPEMAGIC MAGIC_SCOOL PIXEL_DTYPES PIXEL_FIELDSURL)validate_pixelsz cooler.creategrp h5py.Groupchroms pd.DataFrameh5optsdictreturnNonecCst|}tj|dtd}|j d|f|j|d||j d|ft|dd|t|}dD]}| |q4|rGt |||dSdS) a4 Write the chromosome table. Parameters ---------- grp : h5py.Group Group handle of an open HDF5 file with write permissions. chroms : DataFrame Chromosome table containing at least 'chrom' and 'length' columns h5opts : dict HDF5 dataset filter options. namedtypeshaper-datalengthr+r1Nr+)r1) lennparrayrcreate_datasetr-rlistkeysremover )r#r%r'n_chromsnamescolumnscolr?^/var/www/html/software/conda/envs/catlas/lib/python3.10/site-packages/cooler/create/_create.py write_chroms0s0   rATbins chromnames list[str] chrom_as_enumboolc s:t|}t|}tt|t|fdd|dD}|r'tjtfd}nt}z|j d |f||d|} WntyX|rUd}t}|j d |f||d|} nYnw|s`d| j d<|j d|ft |d d||j d|ft |d d|t | } d D]} | | q| rt||| d Sd S)a4 Write the genomic bin table. Parameters ---------- grp : h5py.Group Group handle of an open HDF5 file with write permissions. bins : pandas.DataFrame BED-like data frame with at least three columns: ``chrom``, ``start``, ``end``, sorted by ``chrom`` then ``start``, and forming a complete genome segmentation. The ``chrom`` column must be sorted according to the ordering in ``chroms``. chromnames : sequence of str Contig names. h5opts : dict HDF5 dataset filter options. cg|]}|qSr?r?).0chromidmapr?r@ qzwrite_bins..rIenumr.Fz /chroms/nameZ enum_pathstartendrIrPrQNrI)rP)rQ)r4r(ziprangeh5py special_dtyperr7 ValueErrorattrsrr8r9r:r ) r#rBrCr'rEr;n_bins chrom_ids chrom_dtypeZ chrom_dsetr=r>r?rJr@ write_binsSsf      r]rZintmax_sizer=pd.Index | list[str]dtypespd.Series | dict[str, Any]c Cst|}td||}|j d|dt|f|fd||j d|dt|f|fd|d|vrH|j d |dt|f|fd|dD]}z||WqJty\YqJw|rw|D]}|j|f||t|f|fd|qadSdS) Nbin1_id)r-r/maxshapebin2_idcountrdrfrg)rd)rf)rg) r8minr7r rrr:rXfloat)r#rZr_r=rar'Z init_sizer>r?r?r@prepare_pixelssb     rkfilepathstr grouppathiterableTabular | Iterable[Tabular]lock Any | Nonetuple[int, int]c s>d}d}t|D]\}} t| tjrdd| D} zu|dur$|td|t |dM} | |fdd|D} t | |d} t || D]\} }| || f| | |||| <qM|| 7}d | vrt|| d  7}| Wdn1swYW|dur|q|dur|ww||fS) a Write the non-zero pixel table. Parameters ---------- filepath : str Path to HDF5 output file. grouppath : str Qualified path to destination HDF5 group. columns : sequence Sequence of column names iterable : an iterable object An object that processes and yields binned contacts from some input source as a stream of chunks. The chunks must be either pandas DataFrames or mappings of column names to arrays. h5opts : dict HDF5 filter options. lock : multiprocessing.Lock, optional Optional lock to synchronize concurrent HDF5 file access. rcSsi|]\}}||jqSr?)valuesrHkvr?r?r@ z write_pixels..Nzwriting chunk r+crGr?r?rHr>r#r?r@rLrMz write_pixels..rg) enumerate isinstancepd DataFrameitemsacquireloggerdebugrVFiler4rTresizesumflushrelease)rlrnr=ror'rqnnztotalichunkfwZdsetsnr>Zdsetr?r|r@ write_pixelss:   rr np.ndarrayc Cs`|d}tj|dtd}d}tt|dD]\}}}||||d<|d}q|||d<|S)Nrdrr,ri@B)r5zerosrrTr) r#rZrZbin1 bin1_offsetcurr_valrP_lengthvaluer?r?r@ index_pixels s  rr;c Cs^|d}tj|dtd}d}tt|D]\}}}||||d<|d}q|||d<|S)NrIrr,r)r5rrrTr) r#r;rZr[ chrom_offsetrrPrrr?r?r@ index_binss  rrrcCsD|j dt|ft|d||j dt|ft|d|dS)a Write the indexes. Parameters ---------- grp : h5py.Group Group handle of an open HDF5 file with write permissions. chrom_offset : sequence Lookup table: chromosome ID -> first row in bin table (bin ID) corresponding to that chromosome. bin1_offset : sequence Lookup table: genomic bin ID -> first row in pixel table (pixel ID) having that bin on the first axis. rr.rN)r)r)r7r4rr)r#rrr'r?r?r@ write_indexes s$ rFinfoscoolcCsd|vsJ|sd|vsJ|ddt|di|d<t|d<dt|d<|r8t|d <t |d <nt |d <t |d <t |d <|j |d S) a Write the file description and metadata attributes. Parameters ---------- grp : h5py.Group Group handle of an open HDF5 file with write permissions. info : dict Dictionary, unnested with the possible exception of the ``metadata`` key. ``metadata``, if present, must be JSON-serializable. Required keys ------------- nbins : int number of genomic bins nnz : int number of non-zero pixels nbinsrgenome-assemblyunknownmetadataz creation-datezcooler-z generated-byformatzformat-versionz format-urlN) setdefaultjsondumpsr rnow isoformatr rr rr r!rYupdate)r#rrr?r?r@ write_infoEs     r rename_dictdict[str, str]c Cst|dd}t|}tj||jjtd}|d=|dj d |f|j |d|t|d}t|}t |dj t j rtt|t|}|djj} tjt|fd} |d =z|dj d |f| | d|WdStyt} |dj d |f| | d|YdSwdS) Nr%r+r,z chroms/namer.rBrIrN bins/chromr3rS)r Z set_indexr4r5r6renameindexrtrr7r-r~rZCategoricalDtyper(rTrUcatcodesrVrWrrX) r#rr'r%r;Z new_namesrBrZrKr[r\r?r?r@_rename_chromsjsN     r dict | NonecCsJt|}|d}t|||Wdn1swY|dS)a Substitute existing chromosome/contig names for new ones. They will be written to the file and the Cooler object will be refreshed. Parameters ---------- clr : Cooler Cooler object that can be opened with write permissions. rename_dict : dict Dictionary of old -> new chromosome names. Any names omitted from the dictionary will be kept as is. h5opts : dict, optional HDF5 filter options. rzN) _set_h5optsopenrZ_refresh)Zclrrr'fr?r?r@ rename_chromss   rpd.Series | dict | Nonekwargsdict[str, Any]cCs6d|vr|dur|d}tdt|Std|S)Nr-zUse dtypes= instead of dtype=zReceived both "dtypes" and "dtype" arguments. Please use "dtypes" to provide a column name -> dtype mapping. "dtype" remains as an alias but is deprecated.)popwarningswarn FutureWarningrX)rarr?r?r@_get_dtypes_args  rcCs|i}|dur ||hd}|D]}||vr!td|dq|dd|ddkr6d|vr6d|d<|dd |S) N> compressionZ scaleoffsetZ track_timesshufflecompression_optschunksZ fletcher32re fillvaluezUnknown storage option ''.rgziprrT)rr9rXr)r'resultZavailable_optskeyr?r?r@rs    rc5 Kst|\}}t|}|dur|rdnd}t| } t|tjs#td|r-|dur-tdt||}dD]}||j vrCtd|dq4|durMgd }nt |}d D] }||vr_| d |qS|durit t }n t |}t t }||t||td }zd d lm}Wn ttfyd}Ynwt||rdd|D}t|j }n(t|tjr|f}t|j }nt|t r|f}tdd|Dj }n|}d}|dur|D]}||vr|tvrdnd}t|d|dq|}|dt|d<t|}z|}Wn tyYnwt|\}}tj||dddgd}t|} t|}!t|}"|s;| r;t dd} | sG| sG| sG| rTt!|"| | | | }#t"|#|}t#$||B}$t%&d|d|d |d!kr{d"D] }%|%|$vrx|$|%=qnnz|$'|Wnty|$|=|$'|YnwWdn 1swY|rat|\}&}'t|\}(})t#$|&d#^}*t#$|(d#G}+|*d$|+|)d$<|*d%|+|)d%<|*d&|+|)d&<|*d'|+|)d'<t |(}dD]}|)|q|rt*|+|)d(||Wdn 1s wYWdn 1swYt#$|d#0}$|$|},|,'d)}-|r?|"|"d*d+|"}.n|"|"}.t+|-|"|.|j t |j,| Wdn 1s[wYnbt#$|d#S}$|$|},t%&d,|,'d$}-t-|-|| t%&d-|,'d(}-t.|-||d| |,'d)}-|r|"|"d*d+|"}.n|"|"}.t+|-|"|.|j t |j,| Wdn 1swYt%&d.t/0|d)}/t1||/|j || |\}0}1t#$|d#{}$|$|},t%&d/|,'d0}-t2|,d(|!|"}2t3|,d)|"|0}3t4|-|2|3| t%&d1i}4| durd2nd3|4d4<| dur| nd5|4d6<|r(d7nd8|4d9<|!|4d:<|"|4d;<|1|4d<<|0|4d=<|durE||4d><|durN||4d?<t5|,|4WddS1s_wYdS)@a Create a new Cooler. Deprecated parameters --------------------- chromsizes : Series Chromsizes are now inferred from ``bins``. append : bool, optional Append new Cooler to the file if it exists. If False, an existing file with the same name will be truncated. Default is False. Use the ``mode`` argument instead. dtype : dict, optional Dictionary mapping column names in the pixel table to dtypes. Use the ``dtypes`` argument instead. NawzSecond positional argument must be a pandas DataFrame. Note that the `chromsizes` argument is now deprecated: see documentation for `create`.zWIf the parameter `append_scool` is set, the parameter `scool_root_uri` must be defined.rR Missing column from bin table: 'rrh)rdrfr)Z default_dtyperr?css|]}|VqdSN)compute)rHxr?r?r@ .szcreate..cSsg|] \}}||jfqSr?r,rur?r?r@rL5szcreate..ZStandardZUserz column not found in input: ''rIr2r+r1r=zTCreating a non-symmetric matrix, but `triucheck` was set to True. Changing to False.FCreating cooler at "::"/)r%rBpixelsindexesrzr%rz bins/startzbins/endrBrrrWriting chroms Writing binszWriting pixelszWriting indexesr Writing infofixedvariablebin-typenullbin-sizezsymmetric-upperZsquarez storage-modenchromsrrrrr)6roprealpathrr~rrrXrr=r8insertr(rrrrjdask.dataframe ImportErrorAttributeError to_delayedrrr copyastypeobjectrrTrr4rrr"maprVrrr create_groupr9r:r rkrarAr] posixpathjoinrrrrr)5cool_urirBrr=rarassemblysymmetric_uppermoder' boundscheck triucheckdupcheck ensure_sortedrqappend append_scoolscool_root_urir file_path group_pathr>dtypes_metadask_dfroZ input_columnsZcol_type chromsizesrClengthsr%binsizer;rZ validatorrr+Zsrc_pathZ src_groupZdst_pathZ dst_groupsrcdsth5r#r_targetrZ ncontactsrrrr?r?r@createsJ %                                         $r-1c  sddlmddlm} |}|dt|d<|dur$dd|D}|dur2tt |d}n|d kr8d}t || }t j d k} | rI|rId } n|} g}t jd | |d }||g}t|D]+\}}|j dt|}||td|d|t|||f||dd| q`t|}|| krdkrnnqtjd|tt|td}t jd | |d }||g}t|dd|ddD]C\}}| fdd|||D||d}|j d|d |}||td|d |d|t|||f||dd| q|}n|}| fdd|D||d}td|t|||f|||d| | rN|rN|D]}|jsF|t |j q<~dS)aa Create a Cooler in two passes via an external sort mechanism. In the first pass, a sequence of data chunks are processed and sorted in memory and saved to temporary Coolers. In the second pass, the temporary Coolers are merged into the output. This way the individual chunks do not need to be provided in any particular order. rCooler) CoolerMergerrINcSsg|]}|dvr|qS)>rdrfr?r{r?r?r@rLryz)create_from_unordered..r-ntFz .multi.cool)suffixdeletedirrzWriting chunk z: r)r=rarr,rcg|]}|qSr?r?rHurir r?r@rLrMrzMerging chunks crr?r?rr r?r@rL%rMz Merging into )apir reducer rrrrdirnamerrosr+tempfileNamedTemporaryFilerr}rmrrrr4r5Zlinspacer^sqrtrTclosedcloser:)rrBrr=rarmergebuf delete_temptemp_dir max_mergerr Z is_windowsrZ temp_filestfurisrrrredgesZtf2Zuris2lohiZ chunk_subsetZ final_urisr?r r@create_from_unordereds       "   r(c Cst|}t|\}}zddlm} ddlm} Wnttfy'd} d} Ynwt|| r1|}z| } Wn tyC|j } Ynwt |d} | |} | D]}|| |vrj|set d|dd| ||=qQt|| r|D]<}d}|D]3}|}z|d ur|t| ||||d W|d ur|n |d ur|ww|t|7}q|qtnk|r| D]:}d}||D]1}z|d ur|t| |||i||d W|d ur|n |d ur|ww|t|7}qqn1z|d ur|t| ||d|d W|d ur|n|d ur|wwWd d SWd d SWd d S1s9wYd S) a Append one or more data columns to an existing table. Parameters ---------- cool_uri : str Path to Cooler file or URI to Cooler group. table : str Name of table (HDF5 group). data : dict-like DataFrame, Series or mapping of column names to data. If the input is a dask DataFrame or Series, the data is written in chunks. chunked : bool, optional If True, the values of the data dict are treated as separate chunk iterators of column data. force : bool, optional If True, replace existing columns with the same name as the input. h5opts : dict, optional HDF5 dataset filter options to use (compression, shuffling, checksumming, etc.). Default is to use autochunking and GZIP compression, level 6. lock : multiprocessing.Lock, optional Optional lock to synchronize concurrent HDF5 file access. rr)Seriesr?rzrz' column already exists. z Use --force option to overwrite.N)r&r')rrrrr)rrr~Zto_framer9r=rVrrXrrrr rr4)rtabler0chunkedforcer'rqrrrZ dask_seriesr<rrr+rrrr?r?r@r2s                   $raI columns : sequence of str, optional Customize which value columns from the input pixels to store in the cooler. Non-standard value columns will be given dtype ``float64`` unless overriden using the ``dtypes`` argument. If ``None``, we only attempt to store a value column named ``"count"``. dtypes : dict, optional Dictionary mapping column names to dtypes. Can be used to override the default dtypes of ``bin1_id``, ``bin2_id`` or ``count`` or assign dtypes to custom value columns. Non-standard value columns given in ``dtypes`` must also be provided in the ``columns`` argument or they will be ignored. metadata : dict, optional Experiment metadata to store in the file. Must be JSON compatible. assembly : str, optional Name of genome assembly. ordered : bool, optional [default: False] If the input chunks of pixels are provided with correct triangularity and in ascending order of (``bin1_id``, ``bin2_id``), set this to ``True`` to write the cooler in one step. If ``False`` (default), we create the cooler in two steps using an external sort mechanism. See Notes for more details. symmetric_upper : bool, optional [default: True] If True, sets the file's storage-mode property to ``symmetric-upper``: use this only if the input data references the upper triangle of a symmetric matrix! For all other cases, set this option to False. mode : {'w' , 'a'}, optional [default: 'w'] Write mode for the output file. 'a': if the output file exists, append the new cooler to it. 'w': if the output file exists, it will be truncated. Default is 'w'. Other parameters ---------------- mergebuf : int, optional Maximum number of records to buffer in memory at any give time during the merge step. delete_temp : bool, optional Whether to delete temporary files when finished. Useful for debugging. Default is False. temp_dir : str, optional Create temporary files in a specified directory instead of the same directory as the output file. Pass ``-`` to use the system default. max_merge : int, optional If merging more than ``max_merge`` chunks, do the merge recursively in two passes. h5opts : dict, optional HDF5 dataset filter options to use (compression, shuffling, checksumming, etc.). Default is to use autochunking and GZIP compression, level 6. lock : multiprocessing.Lock, optional Optional lock to control concurrent access to the output file. ensure_sorted : bool, optional Ensure that each input chunk is properly sorted. boundscheck : bool, optional Input validation: Check that all bin IDs lie in the expected range. dupcheck : bool, optional Input validation: Check that no duplicate pixels exist within any chunk. triucheck : bool, optional Input validation: Check that ``bin1_id`` <= ``bin2_id`` when creating coolers in symmetric-upper mode. a Notes ----- If the pixel chunks are provided in the correct order required for the output to be properly sorted, then the cooler can be created in a single step by setting ``ordered=True``. If not, the cooler is created in two steps via an external sort mechanism. In the first pass, the sequence of pixel chunks are processed and sorted in memory and saved to temporary coolers. In the second pass, the temporary coolers are merged into the output file. This way the individual chunks do not need to be provided in any particular order. When ``ordered=False``, the following options for the merge step are available: ``mergebuf``, ``delete_temp``, ``temp_dir``, ``max_merge``. Each chunk of pixels will go through a validation pipeline, which can be customized with the following options: ``boundscheck``, ``triucheck``, ``dupcheck``, ``ensure_sorted``. c sfdd}|S)Ncs|jjdi|_|S)Nr?)__doc__r)funcrr?r@decoratesz#_format_docstring..decorater?)rr0r?r/r@_format_docstrings r1)Zother_parametersnotesrrrlist[str] | Nonedict[str, Any] | Nonerr str | Noneorderedrrrr r!r"rrrrcCst|tjtfrt|ddg}d}|r+t||||||||| ||||||ddSt|||fid|d|d|d|d |d | d |d |d |d|d|d|d| d| d| d| dS)a Create a cooler from bins and pixels at the specified URI. Because the number of pixels is often very large, the input pixels are normally provided as an iterable (e.g., an iterator or generator) of DataFrame **chunks** that fit in memory. .. versionadded:: 0.8.0 Parameters ---------- cool_uri : str Path to cooler file or URI string. If the file does not exist, it will be created. bins : pandas.DataFrame Segmentation of the chromosomes into genomic bins as a BED-like DataFrame with columns ``chrom``, ``start`` and ``end``. May contain additional columns. pixels : DataFrame, dictionary, or iterable of either A table, given as a dataframe or a column-oriented dict, containing columns labeled ``bin1_id``, ``bin2_id`` and ``count``, sorted by (``bin1_id``, ``bin2_id``). If additional columns are included in the pixel table, their names and dtypes must be specified using the ``columns`` and ``dtypes`` arguments. For larger input data, an **iterable** can be provided that yields the pixel data as a sequence of chunks. If the input is a dask DataFrame, it will also be processed one chunk at a time. {other_parameters} See also -------- cooler.create_scool cooler.create.sanitize_records cooler.create.sanitize_pixels {notes} rdrfT) r=rarrrrrrrrr'rqr=rarrrrrrrrr'rqrr r!r"N)r~rrr(Z sort_valuesrr()rrBrr=rarrr6rrrr r!r"rrrrr'rqr?r?r@ create_coolerst>      r7cell_name_pixels_dictdict[str, pd.DataFrame]c, s t|\}}t|}ttjrfdd|D}t|}n2}t|dkr*td|tt |gdt|}t|}t ||D] \}}||krOtdqCt ||}dD]}|j vrftd|dqW|d urpt t}n t |}t t}||d td <t}z|}Wn tyYnwt |\}} tj|| d d d gd}!t}"t|!}#t}$t|| >}%td|d|d|dkrdD] }&|&|%vr|%|&=qnz|%|Wnty|%|=|%|YnwWd n 1swYt|d/}%|%|}'td|'d}(t|(|!|td|'d}(t|(|!d |Wd n 1sBwYt|dO}%|%|}'tdi})|"d ur`dnd|)d<|"d urk|"nd|)d<|#|)d<t||)d <|$|)d!<|d ur||)d"<|d ur||)d#<t|'|)d$Wd n 1swY|D][}*d|*vr|*dd%}+n|*}+t |d&|+||*||*fid'|d(|d#|d)|d*|d+|d,d-d.|d/|d0|d1|d2|d3|d4| d5| d6| d7| d8d$d9|qd S):a Create a single-cell (scool) file. For each cell store a cooler matrix under **/cells**, where all matrices have the same dimensions. Each cell is a regular cooler data collection, so the input must be a bin table and pixel table for each cell. The pixel tables are provided as a dictionary where the key is a unique cell name. The bin tables can be provided as a dict with the same keys or a single common bin table can be given. .. versionadded:: 0.8.9 Parameters ---------- cool_uri : str Path to scool file or URI string. If the file does not exist, it will be created. bins : :class:`pandas.DataFrame` or Dict[str, DataFrame] A single bin table or dictionary of cell names to bins tables. A bin table is a dataframe with columns ``chrom``, ``start`` and ``end``. May contain additional columns. cell_name_pixels_dict : Dict[str, DataFrame] Cell name as key and pixel table DataFrame as value. A table, given as a dataframe or a column-oriented dict, containing columns labeled ``bin1_id``, ``bin2_id`` and ``count``, sorted by (``bin1_id``, ``bin2_id``). If additional columns are included in the pixel table, their names and dtypes must be specified using the ``columns`` and ``dtypes`` arguments. For larger input data, an **iterable** can be provided that yields the pixel data as a sequence of chunks. If the input is a dask DataFrame, it will also be processed one chunk at a time. {other_parameters} See also -------- cooler.create_cooler cooler.zoomify_cooler {notes} csi|]}|qSr?r?)rH cell_namerBr?r@rxsz create_scool..rzAt least one bin must be given.rRz.Bins and pixel dicts do not have matching keysrrNrIr2r+r1rrrrr)r%rBrzrr%rrBrrrrrrrZncellsrrrTrz ::/cells/r=rarr6rrrrrrrr'rqrr r!r"rr)!rrr~rrsortedr4rXnextiterrTrr=r(rrrrrrrrrrVrrrrrAr]rsplitr),rrBr8r=rarrr6rrrr r!r"rrrrr'rqrrrZ bins_dict cell_namesZ bins_keysZkey_binsZ key_pixelsr>rrrCrr%rr;rZrr+rr#rrr:r?r;r@ create_scoolZs C                              rA)r#r$r%r&r'r(r)r*)T) r#r$rBr&rCrDr'r(rErFr)r*)r#r$rZr^r_r^r=r`rarbr'r(r)r*)rlrmrnrmr=r`rorpr'r(rqrrr)rs)r#r$rZr^rr^r)r)r#r$r;r^rZr^r)r) r#r$rrrrr'r(r)r*)F)r#r$rr(rrFr)r*)r#r$rrr'r(r)r*r)rrr'rr)r*)rarrrr)r)r'rr)r)NNNNTNNTTTFNFFN)NNNrTNr )FFNN)NNNNFTrrTNr TTTFNN)*rrmrBr&rrpr=r3rar4rrrr5r6rFrrFrrmrr^r rFr!r5r"rmrrFrrFrrFrrFr'rrqrrr)r*)*rrmrBr&r8r9r=r3rar4rrrr5r6rFrrFrrmrr^r rFr!r5r"r^rrFrrFrrFrrFr'rrqrrr)r*)L __future__rrZos.pathpathrrrrrtypingrrrVnumpyr5ZpandasrZ simplejsonrZ_loggingrZ_typingr_versionr r r corer r utilrrrrrrrrrrrrrrrrrr r!Z_ingestr"rrAr]rkrrrrrrrrrrr(rstripZ_DOC_OTHER_PARAMSZ _DOC_NOTESr1r7rAr?r?r?r@s      <  ( F 3 > % %'   z k a<>  k