tf#XddlZddlZddlZddlZddlmZ GddZ GddZ y)NcleZdZdZdZdZdZdZdZdZ dZ d Z d Z d Z d Zd ZdZdZdZy) ParallelDatac||_yNdata)selfrs W/var/www/html/software/conda/envs/higlass/lib/python3.12/site-packages/clodius/fpark.py__init__zParallelData.__init__ s  c^t|jDcgc] }|| c}Scc}wr)rrr funcds r mapzParallelData.map s#dii8T!W8998s*c,t|jSr)lenrr s r countzParallelData.counts499~r cB|j|jz|_|Srr)r other_datas r unionzParallelData.unionsII /  r cy)Nrs r getNumPartitionszParallelData.getNumPartitionssr cztjt}|jD]}||dxxdz cc<|S)zE Count the number of elements with a particular key. rr)col defaultdictintr)r countsrs r countByKeyzParallelData.countByKeys>% A 1Q4LA L  r c|Srr)r nums r coalescezParallelData.coalesce$s r ctjt}|jD]}||dj |dt |j S)zh When called on a dataset of (K, V) pairs, returns a dataset of (K, Iterable) pairs. rr)rrlistrappendritems)r bucketsrs r groupByKeyzParallelData.groupByKey'sR//$' 'A AaDM 1 & 'GMMO,,r ct||j}tttj j |S)zd Flatten a list of results mapped to the data and return as one large list. )rrrr'itchain from_iterable)r rresults r flatMapzParallelData.flatMap2s3 T499%D!7!7!?@AAr c6|jD] }|| yrrrs r foreachzParallelData.foreach;s A G r c(||jyrrr rs r foreachPartitionzParallelData.foreachPartition?s  TYYr ctjt}|jD]}||dj |dt }|D]}t j|||||< t|jS)aV When called on a dataset of (K, V) pairs, returns a dataset of (K, V) pairs where the values for each key are aggregated using the given reduce function func, which must be of type (V,V) => V. Like in groupByKey, the number of reduce tasks is configurable through an optional second argument. rr) rrr'rr(dictftreducerr))r rr*rreduced_bucketskeys r reduceByKeyzParallelData.reduceByKeyBs//$' 'A AaDM 1 & '& AC#%99T73<#@OC  AO11344r c.tjt}|jD]}||dj |dt }|D]*}|j }||D] } ||| } |||<,t|jS)Nrr) rrr'rr(r8copyrr)) r start_valseq_func comb_funcr*rr;r<comb_valvals r aggregateByKeyzParallelData.aggregateByKeyTs//$' 'A AaDM 1 & '& ,C ~~'Hs| 3#Hc2 3#+OC  , O11344r cBtj||jSr)r9r:rr5s r r:zParallelData.reducebsyytyy))r c|jSrrrs r collectzParallelData.collectes yyr c |jd|Srr)r ns r takezParallelData.takehsyy!}r N)__name__ __module__ __qualname__r rrrrr"r%r+r1r3r6r=rEr:rHrKrr r rrsS:  -B5$ 5*r rcFeZdZdZdZedZedZedZy)FakeSparkContextz6 Emulate a SparkContext for local processing. cyrrrs r r zFakeSparkContext.__init__qs r ct|Sr)rrs r parallelizezFakeSparkContext.parallelizets D!!r c t|d5}tttd|j cdddS#1swYyxYw)z Load a single file as a ParallelData set. :param filename: A file to be loaded line by line @return: A ParallelData object wrapping the lines in the file. rc"|jSr)strip)xs r z1FakeSparkContext.singleTextFile..s 1779r N)openrr'r readlines)filenamefs r singleTextFilezFakeSparkContext.singleTextFilexsD(C  OAS)rns+ aaH)=)=r