o Nrfm @sddlmZddlZddlmZddlmZddlmZddl m Z ddl m Z ddl mZdd lmZmZed d Zdd ed dd dddf ddZdddZddZddZdS)) annotationsN)partial) open_files)concat) from_delayed) read_bytes)delayed) parse_bytessystem_encodingT)ZpureZinferstrictFc s|dur |dur tdt|trt|}|durndvr"} dnd} t|fd|| d|p1i} |durCfdd| D} nqg} td t| |D]} | | | |}tttt t t d |}| |qMnFt |fdurynd |d |d |pi}|dfddtD} rttfddt|dD}ddt| |D} | std||rt| } | S)a\ Read lines from text files Parameters ---------- urlpath : string or list Absolute or relative filepath(s). Prefix with a protocol like ``s3://`` to read from alternative filesystems. To read from multiple files you can pass a globstring or a list of paths, with the caveat that they must all have the same protocol. blocksize: None, int, or str Size (in bytes) to cut up larger files. Streams by default. Can be ``None`` for streaming, an integer number of bytes, or a string like "128MiB" compression: string Compression format like 'gzip' or 'xz'. Defaults to 'infer' encoding: string errors: string linedelimiter: string or None collection: bool, optional Return dask.bag if True, or list of delayed values if false storage_options: dict Extra options that make sense to a particular storage connection, e.g. host, port, username, password, etc. files_per_partition: None or int If set, group input files into partitions of the requested size, instead of one partition per file. Mutually exclusive with blocksize. include_path: bool Whether or not to include the path in the bag. If true, elements are tuples of (line, path). Default is False. Examples -------- >>> b = read_text('myfiles.1.txt') # doctest: +SKIP >>> b = read_text('myfiles.*.txt') # doctest: +SKIP >>> b = read_text('myfiles.*.txt.gz') # doctest: +SKIP >>> b = read_text('s3://bucket/myfiles.*.txt') # doctest: +SKIP >>> b = read_text('s3://key:secret@bucket/myfiles.*.txt') # doctest: +SKIP >>> b = read_text('hdfs://namenode.example.com/myfiles.*.txt') # doctest: +SKIP Parallelize a large file by providing the number of uncompressed bytes to load into each partition. >>> b = read_text('largefile.txt', blocksize='10MB') # doctest: +SKIP Get file paths of the bag by setting include_path=True >>> b = read_text('myfiles.*.txt', include_path=True) # doctest: +SKIP >>> b.take(1) # doctest: +SKIP (('first line of the first file', '/home/dask/myfiles.0.txt'),) Returns ------- dask.bag.Bag or list dask.bag.Bag if collection is True or list of Delayed lists otherwise. See Also -------- from_sequence: Build bag from Python sequence Nz7Only one of blocksize or files_per_partition can be setN  z r rt)modeencodingerrors compressionnewlinec s*g|]}tttttd|qS) delimiter)rlistrfile_to_blocks).0Zfil) include_path linedelimiterV/var/www/html/software/conda/envs/catlas/lib/python3.10/site-packages/dask/bag/text.py ns zread_text..rr F)r blocksizesamplerrcsg|] }tt|qSr)rdecode)rb)rrrrrrscs"g|] \}}|gt|qSr)len)ripath) raw_blocksrrrs"cSsg|] \}}tt||qSr)r attach_path)rentryr(rrrrszNo files found) ValueError isinstancestrr rranger&rrmaprrappendrencoder enumeratezipr)Zurlpathr!rrrrZ collectionZstorage_optionsZfiles_per_partitionrrfilesblocksstartZ block_filesZ block_linesopathsr)rrrrr)r read_texts|H        r;c#sW}dur;|}|sgWdS|}fddfdd|ddD|ddDEdHn|D]}rF|jfn|Vq=WddSWddS1s]wYdS)Nc3s"|] }r |jfn|VqdSN)r(rline)r lazy_filerr s  z!file_to_blocks..cg|]}|qSrrr=rrrrz"file_to_blocks..)readsplitr()rr?rftextpartsr>r)rrr?rrs"  $ "rccs|D]}||fVqdSr<r)blockr(prrrr+s r+csz|||}dvrtj|d}t|S|sgS|}fdd|ddD|s8|dd}|Sg}|S)Nr )rcrArr)rtline_delimiterrrrrBzdecode..rC)r$ioStringIOrrEendswith)rIrrrMrGlinesrHoutrrLrr$s  r$r<) __future__rrN functoolsrZ fsspec.corerZtlzrZ dask.bag.corerZ dask.bytesrZ dask.delayedrZ dask.utilsr r r;rr+r$rrrrs0