DUfddlZddlZddlZddlZddlZddlmZddlmZm Z ddl Z GddZ dZ dZdZd Zd Zd Zd Zd ZdZdZiZdZdZdS)N)Path)PoolRawArrayc"eZdZdZdZddZdS)TempFileHolderc,t|_|SN)listtmpfiles)selfs G/var/www/html/software/conda/lib/python3.11/site-packages/qnorm/util.py __enter__zTempFileHolder.__enter__ s  c|jD]5}tj|rtj|6dSr )r ospathisfileremove)r exc_typeexc_valexc_tbfiles r __exit__zTempFileHolder.__exit__sAM  Dw~~d##  $  rctj}tdD]}dt jt jt jzd}|d|||}tj |rmt| |j||cSdS)Ndr)k/)tempfile gettempdirrangejoinrandomchoicesstringascii_uppercasedigitsrrexistsrtouchr append)r prefixsuffixtmpdirirand_seqfilenames r get_filenamezTempFileHolder.get_filenames$&&s Awwv5 EKKKH!==6=8=V==Hw~~h'' X$$&&& $$X... rN)rr)__name__ __module__ __qualname__rrr2rr rr sF         rrcddl}t|}d|||dddjD}d|||dddgjD}|||fS) zc parse a csv file (memory efficient) and get the columns, index and delimiter from it. rNc,g|]}t|Sr6str.0cols r zparse_csv...s.  Cr #)sepnrowscomment index_colc,g|]}t|Sr6r9)r<rows r r>zparse_csv..4s.     C   r)rArCrDusecols)pandas get_delimread_csvcolumnsindex)infilepd delimiterrKrLs r parse_csvrP&s &!!I;;  SA   G   ;;  3!aS      E E9 $$rcrddl}d||ddjD}||5}t |dksJ|d}||dj}dddn #1swxYwY||fS)zA parse a hdf file and get the columns and index from it. rNcg|]}|Sr6r6r;s r r>zparse_hdf..DsKKKssKKKr)startstoprL)rHread_hdfrKHDFStorelenkeys select_columnvalues)rMrNrKhdfkeyrLs r parse_hdfr^=sLKbkk&kBBJKKKG V  7388::!####hhjjm!!#w//6777777777777777 E>sAB**B.1B.c ddl}ddlm}||}|j j} fdt jD}d|D}t|dksJt|dkr-|d | | gj j }d}n*d tt |jj }d } fd |D}||||fS) zE parse a parquet file and get the columns and index from it. rN) ParquetFilecNg|]!}j|j"Sr6)schemacolumnname)r<col_imetadatas r r>z!parse_parquet..Ws<  u%%*rcg|]}d|v| S)__index_level_r6r;s r r>z!parse_parquet..[s#DDD#,<,C,C#,C,C,CrrU)rKTz__non-existing-col__Fc g|] }|k| Sr6r6)r<r=rDs r r>z!parse_parquet..gs#:::s )9)9s)9)9)9r)rHpyarrow.parquetr`rfrbto_arrow_schemar" num_columnsrX read_parquetrLr[r num_rows) rMrNr`parquetrbrK index_colsrL index_usedrDrfs @@r parse_parquetrrLs9++++++k&!!GH _ , , . .F8/00GEDDDDJ z??a     :!qM  <<BI * U7+45566 ::::g:::G E:v --rc.d|D}t|d5}||dg|zdzt|D]%}|t ||& ddddS#1swxYwYdS)z- glue multiple csv into a single csv c,g|]}t|Sr6 read_linesr<r s r r>zglue_csv..o CCChZ))CCCrwr N)openwriter#zip _glue_csv)outfileheadercolfilesrO open_colfiles lotsaliness r glue_csvrksDC(CCCM gs  "?? @ @C ))SEEKN* + +C u{{}}%% %D $;rc @ddl}d|D}t|D]}|tj|}|ddd|j_||_| d}| |dddd d dS) - glue multiple hdf into a single hdf rNc,g|]}t|Sr6rurws r r>zglue_hdf..rxrTinplacefloat32qnormatable)r]r+modeformat min_itemsize) rHr} DataFramerr set_indexrLrdrKastypeto_hdf)rrrrNrrdfs r glue_hdfrsCC(CCCM=)   \\")J// 0 0 Q %%%  YYy ! !        rcddl}ddl}|j||}d|D}t |djdvr tj} nat |djdvr tj } n+td|djdt|D]} |r`| tj | } | dd | | } d| j_na| tj | | } | d } | | jdd } || _||j| dS)rrNc,g|]}t|Sr6rurws r r>z glue_parquet..rxrrUfloatdoublez The datatype zc is not (yet) supported. Change the dtype of the parquet file, or make an issue on the github page.Trdtype)drop)axis)r)rHrjro ParquetWriterr:fieldtyperrfloat64NotImplementedErrorr}rrrrrLrd reset_indexrrK write_tableTable from_pandas) rrrrqrbrNpyarrowwriterrrrrs r glue_parquetrs _ * *7F ; ;FCC(CCCM 6<<??   W--  V\\!__ ! " "x 0 0 ! FLLOO0      =) @ @  0bi 3344B LLDL ) ) )5!!B BHMMbi 335AABT**BAQ//B !:!:2!>!>???? @ @rc ddl}tj5tjd||dddddjjjj}dddn #1swxYwY|S)NrignoreTir@)rAiteratorrBrCrD) rHwarningscatch_warnings simplefilterrJ_enginerdialectrO)rrNrOs r rIrIs  " "))h'''KK td$q   $wy ))))))))))))))) sAA((A,/A,c#DK|D]}tj|dVdS)a Iterate over lines of a file, multiple lines at the same time. This can be useful when iterating over multiple files at the same time on a slow filesystem (e.g. hard disks). In this case the file can be read on longer batches continuously so the reader does not have to switch as often. Args: file: path to file n: number of lines to read at a time Returns: a list with a string per line T) allow_pickleN)rload)filesrs r rvrvs?//gd.......//rc \ttj||jd|jdz}tj|||j}tj|||t|t|||jf5}tj | tt|jdtjj}dddn #1swxYwY||fS)zI private argsort function of qnorm that works with multiple cpus rrUr) processes initializerinitargsN)rr ctypeslibas_ctypes_typer frombufferreshapecopytorr _worker_initarraymap _worker_sortr"int64T)_arrayncpusr data_sharedrpool sorted_idxs r _parallel_argsortrsG  ##E**FLOfl1o,MK =E 2 2 2 : :6< H HDIdFMM%(()))  udj1     X HH\5A#7#7 8 8                      srsY  ********        4%%%.   ...> < < <      0"@"@"@J///$2 """"""""r