
    >ie,                       d dl mZ d dlZd dlZd dlZd dlZd dlmZ d dlm	Z	 d dl
Z
d dlZd dlmZ d dlmZ d dlmZ d dlmZmZ d d	lmZ d d
lmZ d dlmZ d dlmZ  eed          ZdddZddddZd Zd Z d Z!d Z"d Z#d Z$d Z%ej&        '                    dddg          d              Z(d! Z)d" Z*d# Z+ej&        ,                    ej-        d$k    d%&          d'             Z.d( Z/d) Z0d* eD             d+ eD             z   Z1ej&        '                    d,e1          d-             Z2d. Z3ej&        '                    d/d0d1g          d2             Z4ej&        '                    d3d4d5g          ej&        '                    d6 e5e                    d7                         Z6d8 Z7d9 Z8ej&        j9        d:             Z:ej&        '                    d;de;fd<ej;        fg          d=             Z<d> Z=d? Z>d@ Z?dS )A    )annotationsN)partial)sleep)compr)
open_files)LocalFileSystem)concatvalmap)compute)
read_bytes)compress)	filetextssync)	schedulers   {"amount": 100, "name": "Alice"}
{"amount": 200, "name": "Bob"}
{"amount": 300, "name": "Charlie"}
{"amount": 400, "name": "Dennis"}
s   {"amount": 500, "name": "Alice"}
{"amount": 600, "name": "Bob"}
{"amount": 700, "name": "Charlie"}
{"amount": 800, "name": "Dennis"}
).test.accounts.1.jsonz.test.accounts.2.jsons   a,b
1,2
s   a,b
3,4
s   a,b
5,6
)z.test.fakedata.1.csvz.test.fakedata.2.csvzsubdir/.test.fakedata.2.csvc                    t          j        t          j                            |                                                     S N)pathlibPathospathabspathas_uri)r   s    ;lib/python3.11/site-packages/dask/bytes/tests/test_local.pyto_urir   /   s,    <--..55777    c                     t          j        t                    5  t          h d           d d d            d S # 1 swxY w Y   d S )N>   allowed.csvsets/are.csvshould/not/be.csvunordered/so/they.csv)pytestraises	TypeErrorr    r   r   test_unordered_urlpath_errorsr&   3   s    	y	!	! 
 
  	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
s   9= =c                 .   t          t          d          5  t          d          \  } }t          | t                    sJ | d d         t          t          t                    d                  d d         k    sJ |                     d          sJ t          |t          t          f          sJ t          |d         t          t          f          sJ t          |d         d         d          sJ t          t          t          |                    t          t                    k    sJ t          t          |           }t          |          t          t                                                    k    sJ 	 d d d            d S # 1 swxY w Y   d S )Nbmode.test.accounts.*   r      
dask)r   filesr   
isinstancebytessortedendswithlisttuplehasattrsummaplenr   r	   setvalues)sampler;   resultss      r   test_read_bytesr>   @   s   	5s	#	#	# 3 3#$677&%(((((bqbzU6%==#34RaR88888u%%%%%&4-00000&)dE]33333vay|V,,,,,3sF##$$E

22226&>>*7||s5<<>>22222223 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3s   E%F

FFc                 n   t          t          d          5  t          ddd          \  } }|                     d          sJ t          ddd          \  } }|                     d          sJ t          ddd          \  } }|                     d          sJ 	 d d d            d S # 1 swxY w Y   d S )	Nr(   r)   r+   P   r-   )r<   	delimiterr      )r   r/   r   r3   r<   r;   s     r    test_read_bytes_sample_delimiterrD   P   s   	5s	#	#	# & &#$6rUSSSu%%%%%#$;BRWXXXu%%%%%#$;AQVWWWu%%%%%%& & & & & & & & & & & & & & & & & &s   BB**B.1B.c                     t          t          d          5  t          dd          \  } }t          |           dk    sJ 	 d d d            d S # 1 swxY w Y   d S )Nr(   r)   r+   z40 Br<   (   )r   r/   r   r9   rC   s     r   test_parse_sample_bytesrH   Z   s    	5s	#	#	# ! !#$6vFFF6{{b     ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! !s   *AAAc                     t          t          d          5  t          dd          \  } }| du sJ 	 d d d            d S # 1 swxY w Y   d S )Nr(   r)   r   FrF   )r   r/   r   )r<   _s     r   test_read_bytes_no_samplerK   `   s    	5s	#	#	#  6uEEE	                 s   A  AAc                    t          t          d          5  t          dd           \  } }t          t	          t
          |                    t          t                    k    sJ 	 d d d            d S # 1 swxY w Y   d S )Nr(   r)   r+   	blocksize)r   r/   r   r7   r8   r9   rC   s     r   test_read_bytes_blocksize_nonerO   f   s    	5s	#	#	# 3 3#$6$GGG3sF##$$E

222223 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3s   AA44A8;A8rN   g      @z5 Bc                   t          t          d          5  t          d|           \  }}t          t	          |           }d                    |                              d          }d                    t                                                                        d          }t          |          t          |          k    sJ 	 d d d            d S # 1 swxY w Y   d S )Nr(   r)   .test.account*rM   r   r-   )	r   r/   r   r   r	   joinsplitr;   r:   )rN   r<   valsr=   ourlines	testliness         r   test_read_bytes_blocksize_typesrW   l   s   	5s	#	#	# / /!"2iHHH6$<<(88G$$**511HHU\\^^,,22599	8}}I...../ / / / / / / / / / / / / / / / / /s   B4CC Cc                     t          t          d          5  t          j        t                    5  t          dd           d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nr(   r)   rQ   g      @rM   )r   r/   r"   r#   r$   r   r%   r   r   $test_read_bytes_blocksize_float_errsrY   v   s    	5s	#	#	# 8 8]9%% 	8 	8'37777	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	88 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8s4   A'AA'A	A'A	A''A+.A+c                     t          t          d          5  t          dd          \  } } }d |D             t                                          k    sJ 	 d d d            d S # 1 swxY w Y   d S )Nr(   r)   r+   T)include_pathc                X    h | ]'}t           j                            |          d          (S    )r   r   rS   ).0r   s     r   	<setcomp>z/test_read_bytes_include_path.<locals>.<setcomp>   s+    9994d##A&999r   )r   r/   r   keys)rJ   pathss     r   test_read_bytes_include_pathrc   |   s    	5s	#	#	# J J !3$GGG1e995999UZZ\\IIIIIJ J J J J J J J J J J J J J J J J Js   ?A$$A(+A(c                 &   t          t          d          5  t          d          dz   } t          | d           \  }}t	          t          t          |                    t          t                    k    sJ 	 d d d            d S # 1 swxY w Y   d S )Nr(   r)   z.test.accounts.*rM   )r   r/   r   r   r7   r8   r9   urlr<   r;   s      r   test_with_urlsrh      s    	5s	#	#	# 3 3&''#-#C48883sF##$$E

22222	3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3s   A!BB
B
win32z!pathlib and moto clash on windows)reasonc                    t          t          d          5  t          j        d          } t	          | d           \  }}t          t          t          |                    t          t                    k    sJ 	 d d d            n# 1 swxY w Y   t          j	        t                    5  t          j        d          } t	          | d            d d d            d S # 1 swxY w Y   d S )Nr(   r)   z./.test.accounts.*rM   zfile://.test.accounts.*)r   r/   r   r   r   r7   r8   r9   r"   r#   OSErrorrf   s      r   test_with_pathsrm      sU   	5s	#	#	# 3 3l/00#C48883sF##$$E

222223 3 3 3 3 3 3 3 3 3 3 3 3 3 3 
w		 ( (l4553$''''( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( (s$   A#BBB+&CC"%C"c                    t          t          d          5  dD ]Nt          d          \  } }t          t	          t
          |                    fdt                                          D             k    sJ t          t          |           }t          d |D                       t          d t                                          D                       k    sJ d	
                    |                              d
          }d	
                    t                                                                        d
          }t          |          t          |          k    sJ P	 d d d            d S # 1 swxY w Y   d S )Nr(   r)   )r,      -   i  rQ   rM   c                P    g | ]"}t          t          |          z  d           #S r]   )maxr9   )r_   vbss     r   
<listcomp>z)test_read_bytes_block.<locals>.<listcomp>   s:     , , ,+,SVVr\A&&, , ,r   c              3  4   K   | ]}t          |          V  d S r   r9   r_   rs     r   	<genexpr>z(test_read_bytes_block.<locals>.<genexpr>   s(      //!s1vv//////r   c              3  4   K   | ]}t          |          V  d S r   rw   r_   rs   s     r   rz   z(test_read_bytes_block.<locals>.<genexpr>   s(      6V6V!s1vv6V6V6V6V6V6Vr   r   r-   )r   r/   r   r4   r8   r9   r;   r   r	   r7   rR   rS   r:   )r<   rT   r=   rU   rV   rt   s        @r   test_read_bytes_blockr}      s   	5s	#	#	# 3 3# 	3 	3B%&6"EEELFDC'' , , , ,05, , ,     vd||,G//w/////36V6Vu||~~6V6V6V3V3VVVVVxx((..u55H0066u==Ix==C	NN22222	33 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3s   EE99E= E=c                    t          t          d          5  dD ]} t          d| d          \  }}t          d| d          \  }}d t          |          D             d	 t          |          D             k    sJ t	          t          |           }d
 |D             }t          d |D                       sJ d                    |                              d          }d                    d t          t                    D                                           d          }||k    sJ d}t          d| |          \  }}t	          t          |           }d |D             }t          d |D                       t          |          dz
  k    sJ d                    |          }	d                    d t          t                    D                       }
|	|
k    sJ 	 d d d            d S # 1 swxY w Y   d S )Nr(   r)   )r,   ro   rp   z1.5 kBz.test.accounts*r-   )rN   rA   s   fooc                    g | ]	}|j         
S r%   key)r_   as     r   ru   z-test_read_bytes_delimited.<locals>.<listcomp>   s    222aAE222r   c                    g | ]	}|j         
S r%   r   r_   r(   s     r   ru   z-test_read_bytes_delimited.<locals>.<listcomp>   s    6V6V6Vqu6V6V6Vr   c                    g | ]}||S r%   r%   rx   s     r   ru   z-test_read_bytes_delimited.<locals>.<listcomp>       ++++1+++r   c              3  @   K   | ]}|                     d           V  dS )r-   Nr3   rx   s     r   rz   z,test_read_bytes_delimited.<locals>.<genexpr>   s.      66Qqzz%((666666r   r   c              3  0   K   | ]}t           |         V  d S r   r/   r_   ks     r   rz   z,test_read_bytes_delimited.<locals>.<genexpr>   s&       A Aaq A A A A A Ar      }c                    g | ]}||S r%   r%   rx   s     r   ru   z-test_read_bytes_delimited.<locals>.<listcomp>   r   r   c              3  @   K   | ]}|                     d           V  dS )r   Nr   rx   s     r   rz   z,test_read_bytes_delimited.<locals>.<genexpr>   s.      55Aqzz$''555555r   rB   c              3  0   K   | ]}t           |         V  d S r   r   r|   s     r   rz   z,test_read_bytes_delimited.<locals>.<genexpr>   s&      <<E!H<<<<<<r   )r   r/   r   r	   r   allrR   rS   r2   r7   r9   )rt   rJ   r;   values2r=   resrU   rV   dourstests              r   test_read_bytes_delimitedr      sZ   	5s	#	#	#    ' 	  	 B"#4eTTTIAv#$5vVVVJAw226&>>2226V6VfWoo6V6V6VVVVVvf~~.G++g+++C66#66666666xx}}**511H A A6%== A A AAAGGNNIy(((( A"#4aPPPIAvvf~~.G++g+++C5555555SAEEEE88C==D88<<fUmm<<<<<D4<<<<<+	                                    s   GG44G8;G8c                    g | ]}|d fS r   r%   r_   fmts     r   ru   ru      s    	'	'	'#3+	'	'	'r   c                    g | ]}|d fS )
   r%   r   s     r   ru   ru      s    *F*F*FC9*F*F*Fr   zfmt,blocksizec                   | t           vrt          j        d           t          t           |          t                    }t          |d          5  | rT|rRt          j        t                    5  t          d|d|            d d d            n# 1 swxY w Y   	 d d d            d S t          d|d|           \  }}|d d         t          t          t                    d                  d d         k    sJ |
                    d          sJ t          t          |           }d	                    |          d	                    d
 t          t                    D                       k    sJ 	 d d d            d S # 1 swxY w Y   d S )N!compression function not providedr(   r)   z.test.accounts.*.jsonr-   )rN   rA   compressionr,   r   r   c                (    g | ]}t           |         S r%   r   r   s     r   ru   z$test_compression.<locals>.<listcomp>   s    -N-N-N1eAh-N-N-Nr   )r   r"   skipr
   r/   r   r#   
ValueErrorr   r2   r3   r   r	   rR   )r   rN   files2r<   r;   r=   s         r   test_compressionr      sK   
(7888HSM5))F	6	$	$	$ P P 	9 	z**  +'# #	                  P P P P P P P P $#	
 
 
 bqbzU6%==#34RaR88888u%%%%%6&>>*xx  CHH-N-Nu-N-N-N$O$OOOOOO)P P P P P P P P P P P P P P P P P Ps=   E<-BE<B	E<B	E<&CE<<F F c                    t          t          d          5  t          d          } t          |           t          t                    k    sJ t	          | t          t                              D ]G\  }}|5 }|                                }|t          |         k    sJ 	 d d d            n# 1 swxY w Y   H	 d d d            d S # 1 swxY w Y   d S Nr(   r)   r+   )r   r/   r   r9   zipr2   read)myfiles	lazy_file	data_filefxs        r   test_open_filesr      sL   	5s	#	#	# - -/007||s5zz))))$'$?$? 	- 	- Iy -aFFHHE),,,,,,- - - - - - - - - - - - - - -	-- - - - - - - - - - - - - - - - - -s6   A C7(B, C,B00C3B04CC
C
encodingzutf-8asciic                    t          t          d          5  t          dd           }t          |          t          t                    k    sJ g }|D ]C}|5 }|                    |                                           d d d            n# 1 swxY w Y   Dt          |           fdt          t                    D             k    sJ 	 d d d            d S # 1 swxY w Y   d S )Nr(   r)   r+   rt)r*   r   c                P    g | ]"}t           |                                       #S r%   )r/   decode)r_   r   r   s     r   ru   z-test_open_files_text_mode.<locals>.<listcomp>   s)    OOOAeAhooh77OOOr   )r   r/   r   r9   appendr   r4   r2   )r   r   datafiler   s   `    r   test_open_files_text_moder      s|   	5s	#	#	# P P/dXNNN7||s5zz)))) 	& 	&D &AFFHH%%%& & & & & & & & & & & & & & &DzzOOOOOOOOOOOOP P P P P P P P P P P P P P P P P Ps6   AC(BCBCB7CC"Cr*   r   rbr   c                   |t           vrt          j        d           t          t           |         t                    }t          |d          5  t          d| |          }g }|D ]C}|5 }|                    |                                           d d d            n# 1 swxY w Y   Dd t          t                    D             }| dk    rd |D             }t          |          |k    sJ 	 d d d            d S # 1 swxY w Y   d S )	Nr   r(   r)   r+   )r*   r   c                (    g | ]}t           |         S r%   r   r   s     r   ru   z/test_open_files_compression.<locals>.<listcomp>  s    ///AuQx///r   r   c                6    g | ]}|                                 S r%   )r   r   s     r   ru   z/test_open_files_compression.<locals>.<listcomp>  s     +++!188::+++r   )r   r"   r   r
   r/   r   r   r   r   r2   r4   )r*   r   r   r   r   r   r   sols           r   test_open_files_compressionr      s    (7888HSM5))F	6	$	$	$ 	! 	!/dLLL 	& 	&D &AFFHH%%%& & & & & & & & & & & & & & &/////4<<++s+++CDzzS     	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	!s7   C=*(BC=B""C=%B"&A	C==DDc                 
   t          t          d          5  t          t          fD ]C} t	          j        t                    5   | dd          \  }}d d d            n# 1 swxY w Y   D	 d d d            d S # 1 swxY w Y   d S )Nr(   r)   r+   z	not-found)r   )r   r/   r   r   r"   r#   r   )funcr<   r;   s      r   test_bad_compressionr     s9   	5s	#	#	# S S, 	S 	SDz** S S!%&8k!R!R!RS S S S S S S S S S S S S S S	SS S S S S S S S S S S S S S S S S Ss5   *A8AA8A""A8%A"&A88A<?A<c                     d} t          j        t          t          f|           5  t	          |            d d d            d S # 1 swxY w Y   d S )Nz
not-a-file)match)r"   r#   FileNotFoundErrorrl   r   )fns    r   test_not_foundr     s    	B	)732	>	>	>  2                 s   AA	Ac                    t          t          d          5  t          d          \  } }t          d          \  } }t          t	          |                    }t          t	          |                    }d |D             d |D             k    sJ t          d           t          D ]?}t          |d          5 }|                    d           d d d            n# 1 swxY w Y   @t          d          \  } }t          t	          |                    }d	 |D             d
 |D             k    sJ 	 d d d            d S # 1 swxY w Y   d S )Nr(   r)   r+   c                    g | ]	}|j         
S r%   _keyr_   aas     r   ru   ztest_names.<locals>.<listcomp>      $$$B$$$r   c                    g | ]	}|j         
S r%   r   )r_   bbs     r   ru   ztest_names.<locals>.<listcomp>      (=(=(=R(=(=(=r   r^   ab   xc                    g | ]	}|j         
S r%   r   r   s     r   ru   ztest_names.<locals>.<listcomp>%  r   r   c                    g | ]	}|j         
S r%   r   )r_   ccs     r   ru   ztest_names.<locals>.<listcomp>%  r   r   )r   r/   r   r4   r	   r   openwrite)rJ   r   r(   r   r   cs         r   
test_namesr     s   	5s	#	#	# > >,--1,--1OOOO$$!$$$(=(=1(=(=(=====a 	 	Bb$ 1               ,--1OO$$!$$$(=(=1(=(=(======> > > > > > > > > > > > > > > > > >s7   B!D>8CD>CD>!C"AD>>EEcompression_openergzipc                *   |\  }}t          |           } t          | dd|          }t          |          dk    sJ d |D             dhk    sJ |D ]1}|5 }|                    d           d d d            n# 1 swxY w Y   2t	          t          j        |                     }|ddgk    sJ  |t
          j                            | |d                   d	          5 }|	                                }d d d            n# 1 swxY w Y   |dk    sJ d S )
NrB   wb)numr*   r   c                    h | ]	}|j         
S r%   r)   )r_   r   s     r   r`   z(test_open_files_write.<locals>.<setcomp>.  s    """qAF"""r   s   000z0.partz1.partr   r   )
strr   r9   r   r2   r   listdirr   rR   r   )tmpdirr   r   openerr/   filr   r   s           r   test_open_files_writer   (  s   ,K[[Fv14[IIIEu::????""E"""tf,,,,   	AGGFOOO	 	 	 	 	 	 	 	 	 	 	 	 	 	 	2:f%%&&EXx(((((	VU1X..	5	5 FFHH              ;;;;;;s$   A88A<	?A<	D  DDc           	     X   t          |           } t          t          d          5  t          d          }t	          j        t	          j        |                    }t          ||          D ]\  }}|j        |j        k    sJ t          |j
                  t          |j
                  k    sJ |5 }|5 }|                                |                                k    sJ 	 d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   	 d d d            d S # 1 swxY w Y   d S r   )r   r   r/   r   cloudpickleloadsdumpsr   r   typefsr   )r   r   myfiles2r   f2f_openf2_opens          r   test_pickability_of_lazy_filesr   :  s   [[F	5s	#	#	# 7 7/00$[%6w%?%?@@(++ 	7 	7EAr6RW$$$$::be,,,, 7fb 7G{{}}666667 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7	7	7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7sZ   BD1D4-C.	"D.C22D5C26D9DD		DD	DD#&D#c                N   t          | dz            }t          j        |d          5 }|                    d           d d d            n# 1 swxY w Y   t	          |dd          }|d         5 }t          d	 |D                       sJ 	 d d d            d S # 1 swxY w Y   d S )
Nzmyfile.txt.gzr   r)   s   hello
worldr   r   )r   r*   r   c              3  @   K   | ]}t          |t                    V  d S r   )r0   r   )r_   lines     r   rz   z'test_py2_local_bytes.<locals>.<genexpr>P  s,      77T:dC((777777r   )r   r   r   r   r   r   )r   r   r   r/   s       r   test_py2_local_bytesr   H  s4   	Vo%	&	&B	2D	!	!	! !Q	   ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! rvD999E	q 8Q77Q7777777778 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8s#   AAA0BB!Bc                l   t          |           } t          j                    }t          j        |            t	          dd          5 }|                    d           d d d            n# 1 swxY w Y   t                                          d          }t          |          dk    sJ d|d         v sJ d|d         v sJ t                      }t          j        |           |                    |d         d          5 }|	                                }d d d            n# 1 swxY w Y   |dk    sJ d S )	Ntmpwhire   r^   /r   ry   )
r   r   getcwdchdirr   r   r   globr9   r   )r   herer   outr   r   s         r   test_abs_pathsr   S  s   [[F9;;DHV	eS		 Q	              



 
 
%
%Cs88q====#a&====CF????			BHTNNN	Q		 ffhh              $;;;;;;s$   A))A-0A- D!!D%(D%)@
__future__r   r   r   r   sys	functoolsr   timer   r   r"   fsspec.compressionr   fsspec.corer   fsspec.implementations.localr   tlzr	   r
   r.   r   dask.bytes.corer   dask.bytes.utilsr   
dask.utilsr   r/   	csv_filesr   r&   r>   rD   rH   rK   rO   markparametrizerW   rY   rc   rh   skipifplatformrm   r}   r   fmt_bsr   r   r   r4   r   r   r   slowr   r   r   r   r   r   r%   r   r   <module>r     s   " " " " " "  				  



                  $ $ $ $ $ $ " " " " " " 8 8 8 8 8 8               & & & & & & % % % % % %            
''V
,
,
,	/	/	 	" /.$5 	8 8 8

 

 

3 3 3 & & &! ! !  3 3 3 sEl33/ / 43/8 8 8J J J3 3 3 CLG+4WXX( ( YX(3 3 3      4 
(	'	'	'	'*F*F*F*F*F	F &11P P 21P6- - - gw%788P P 98P $..U,,! ! -, /.! S S S   > > >& -tvty>Q/RSS  TS"7 7 78 8 8    r   