
    >ieO                    b   d dl mZ d dlZd dlZd dlZd dlmZmZ d dlm	Z	 d dl
Z
 e
j        d          Z e
j        d          Zd dlZd dlmZ d dlmZ d d	lmZmZ d dlZd d
lmZ d dlmZ d dlmZ d dlmZ d dlm Z m!Z!m"Z" d dl#m$Z$m%Z%m&Z&m'Z'm(Z( d dl)m*Z* d dl+m,Z,m-Z-m.Z.m/Z/ d dl0m1Z1 d dl2m3Z3m4Z4m5Z5m6Z6 d dl7m8Z8 d eD             dgz   Z9d Z:d Z;d<                                Z=e=>                    dd          Z?d<                                Z@d<                                ZAdddd ZBd! eBC                                D             ZDd"d#d$d ZEeBfd%ZFd& ZGd'ZHd(ZId)ZJeJ>                    d*d+          ZKe
jL        M                    d,ejN        eBfejO        eDfejP        eEfg          ZQeQd-             ZReQd.             ZSeQd/             ZTeQd0             ZUeQd1             ZVeQd2             ZWeQd3             ZXe
jL        M                    d4ejN        ejN        eBfejO        ejO        eDfg          d5             ZYe
jL        M                    d4ejN        ejN        eBfejO        ejO        eDfg          d6             ZZe
jL        M                    d4ejN        ejN        eBfejO        ejO        eDfg          d7             Z[e
jL        M                    d8ejN        ejN        eBeJfejO        ejO        eDeKfg          d9             Z\d:d;gd:d;ggZ]d<d=gd<d=ggZ^e
jL        M                    d>ejN        e]fejO        e^fg          d?             Z_e
jL        M                    d>ejN        e]fejO        e^fg          d@             Z`e
jL        M                    dAejN        ejN        e=dfejO        ejO        e?dfejO        ejO        e@dBfg          dC             Zae
jL        b                    e! dDE          dF             Zce
jL        M                    dGejN        ejN        e=dHfejO        ejO        e?dIdJgfg          dK             Zde
jL        M                    dGejN        ejN        e=dHfejO        ejO        e?dIdLgfg          dM             Zee
jL        M                    d4ejN        ejN        eBfejO        ejO        eDfg          dN             Zfe
jL        M                    d4ejN        ejN        eBfejO        ejO        eDfg          dO             Zge
jL        M                    dPejN        eBfejO        eDfg          dQ             Zhe
jL        M                    dPejN        eBfejO        eDfg          dR             Zie
jL        M                    dPejN        eBfejO        eDfg          dS             Zje
jL        M                    dPejN        eBfejO        eDfg          dT             Zke
jL        M                    dPejN        eBfejO        eDfg          dU             ZldV ZmdW ZndX ZodY ZpdZ Zqd[ Zrd\ Zsd] Ztd^ Zue
jL        jv        e
jL        M                    d_d`dag          db                         Zwdc Zxdd Zyde Zze
jL        M                    dfddgg          e
jL        M                    dhe9          di                         Z{e
jL        j|        dj             Z}dk Z~e
jL        M                    dlg dm          dn             Zdo Zdp Zdq Zdr Zds Zdt Zdu Zdv Zdw Zdx Zdy Zdz Zd{ Zd| Zd} Zd~ Ze
jL                            dE          Ze
jL        M                    d e
j        de           e
j        de          dg          d             Zd Zd Zd Zd Zd Zd Ze
jL        b                    e  dE          d             Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Ze
jL                            dE          d             Ze
jL        b                     eej                   ed          k    dE          d             Zd Zd Zd Zd Zd Zd Zd Zd Ze
jL        M                    dddg          d             Ze
jL        M                    dddddddgdddfddgdddfg          d             Zd Ze
jL        M                    ddIdgdgg dgg g g dIgg dIggd  edg          D             g          d             Zd Zd Zd Ze
jL        j        d             Zd ZdÄ ZdĄ Ze
jL        M                    dddg          dƄ             ZdS )    )annotationsN)BytesIOStringIO)mockpandaszdask.dataframe)compr)Version)partition_allvalmap)compute_as_if_collection)
read_bytes)compress)flatten)PANDAS_GE_140PANDAS_GE_200tm)_infer_block_sizeauto_blocksize
block_maskpandas_read_texttext_blocks_to_pandas)optimize_dataframe_getitem)	assert_eqget_string_dtypehas_known_categoriespyarrow_strings_enabled)DataFrameIOLayer)filetext	filetextstmpdirtmpfile)	hlg_layerc                    g | ]}|S  r$   ).0fmts     @lib/python3.11/site-packages/dask/dataframe/io/tests/test_csv.py
<listcomp>r(   ,   s    )))CC)))    c                    d                     t          t          j        |                                                     d                              S )N
)joinmapstrstripsplit)ss    r'   normalize_textr2   /   s4    99SAGGIIOOD$9$9::;;;r)   c                L    t           j                            |           d         S )N   )ospathr0   )r6   s    r'   parse_filenamer7   3   s    7==q!!r)   z
name,amount
Alice,100
Bob,-200
Charlie,300
Dennis,400
Edith,-500
Frank,600
Alice,200
Frank,-200
Bob,600
Alice,400
Frank,200
Alice,300
Edith,600
,	z
name   amount
Alice    100
Bob     -200
Charlie  300
Dennis   400
Edith   -500
Frank    600
Alice    200
Frank   -200
Bob      600
Alice    400
Frank    200
Alice    300
Edith    600
a  
Date,Open,High,Low,Close,Volume,Adj Close
2015-08-28,198.50,199.839996,197.919998,199.240005,143298900,199.240005
2015-08-27,197.020004,199.419998,195.210007,199.160004,266244700,199.160004
2015-08-26,192.080002,194.789993,188.369995,194.679993,328058100,194.679993
2015-08-25,195.429993,195.449997,186.919998,187.229996,353966700,187.229996
2015-08-24,197.630005,197.630005,182.399994,189.550003,478672400,189.550003
2015-08-21,201.729996,203.940002,197.520004,197.630005,328271500,197.630005
2015-08-20,206.509995,208.289993,203.899994,204.009995,185865600,204.009995
2015-08-19,209.089996,210.009995,207.350006,208.279999,167316300,208.279999
2015-08-18,210.259995,210.679993,209.699997,209.929993,70043800,209.929993
s3   name,amount,id
Alice,100,1
Bob,200,2
Charlie,300,3
s   name,amount,id
s4   name,amount,id
Dennis,400,4
Edith,500,5
Frank,600,6
)2014-01-01.csv2014-01-02.csv2014-01-03.csvc                B    i | ]\  }}||                     d d          S )   ,   	)replace)r%   kvs      r'   
<dictcomp>rC   r   s,    GGG6AqQ		$&&GGGr)   sT       name  amount  id
   Alice     100   1
     Bob     200   2
 Charlie     300   3
s       name  amount  id
sT       name  amount  id
  Dennis     400   4
   Edith     500   5
   Frank     600   6
c                    t                               d t          |           D                       }|                    t	                      t
          t
          d          }|S )Nc                r    g | ]4}t                               t          t          |                             5S r$   )pdread_csvr   	csv_files)r%   rA   s     r'   r(   zread_files.<locals>.<listcomp>   s0    SSS1BKK	! 5 566SSSr)   nameamountidrF   concatsortedastyper   int)
file_namesdfs     r'   
read_filesrT      sQ    	SSz@R@RSSS	T	TB	,..#SII	J	JBIr)   c                    t                               fdt          |           D                       }|                    t	                      t
          t
          d          }|S )Nc                "    g | ]} |fi S r$   r$   )r%   nhandlerkwargss     r'   r(   z#read_files_with.<locals>.<listcomp>   s+    EEEQGGA((((EEEr)   rI   rM   )rR   rX   rY   rS   s    `` r'   read_files_withrZ      s\    	EEEEE&2D2DEEE	F	FB	,..#SII	J	JBIr)   sL   # some header lines
# that may be present
# in a data file
# before any datasB   # some footer lines
# that may be present
# at the end of the files   str, int, int
r>   r?   zreader,filesc                    |d         }t          | |di           }t          |j                  g dk    sJ t          |          dk    sJ |j                                        dk    sJ d S )Nr:   r)   rI         )r   listcolumnslenrL   sumreaderfilesbrS   s       r'   test_pandas_read_textrf      sv    A	&!S"	-	-B
7777777r77a<<<<599;;)######r)   c                z    |d         }t          | |ddddgi          }t          |j                  ddgk    sJ d S )Nr:   r)   usecolsrJ   rL   )r   r^   r_   rb   s       r'   test_pandas_read_text_kwargsri      sN    A	&!S9vtn*E	F	FB
~------r)   c                d    |d         }t          | |di ddi          }|j        j        dk    sJ d S )Nr:   r)   rK   float)r   rK   dtyperb   s       r'   $test_pandas_read_text_dtype_coercionrm      sB    A	&!S"x.A	B	BB9?g%%%%%%r)   c                   |d         }|                     dd          \  }}|dz   }t          | ||i           }t          |j                  g dk    sJ t	          |          dk    sJ |j                                        dk    sJ d S )Nr:      
r4   rI   r\   r]   )r0   r   r^   r_   r`   rL   ra   )rc   rd   re   headerrS   s        r'   !test_pandas_read_text_with_headerrq      s    Aq!!IFAe^F	&!VR	0	0B
7777777r77a<<<<599;;)######r)   c                t   fdt                    D             }i }t          | d         di           }d                             d          d         dz   }t          | ||||          }t	          |t
          j                  sJ t          |j                  g dk    sJ t          | ||||          }t	          |t
          j                  sJ t          |d          sJ t                      rt          |j                  dk    snnJ t          |j                                        d
           d S )Nc                "    g | ]}|         gS r$   r$   r%   rA   rd   s     r'   r(   z5test_text_blocks_to_pandas_simple.<locals>.<listcomp>   s    000QuQxj000r)   r:   r)   ro   r   rI   daskr]   r\   i4  )rO   r   r0   r   
isinstancedd	DataFramer^   r_   hasattrr   r`   ru   r   rK   ra   )rc   rd   blocksrY   headrp   rS   valuess    `      r'   !test_text_blocks_to_pandas_simpler}      sB   0000&--000FFFE*:$;S"EED#$**511!4u<F	vvvtV	D	DBb",'''''
7777777"6664HHFfbl+++++66"""""$;$=$=D3v{q      bimmoo@AAAAAr)   c                   fdt                    D             }d |D             }dddgi}t          | d         d|          }d                             d          d	         dz   }t          | ||||          }t	          |j                  ddgk    sJ |                                }|j        |j        k                                    sJ d S )
Nc                     g | ]
}|         S r$   r$   rt   s     r'   r(   z5test_text_blocks_to_pandas_kwargs.<locals>.<listcomp>   s    ...1eAh...r)   c                    g | ]}|gS r$   r$   )r%   re   s     r'   r(   z5test_text_blocks_to_pandas_kwargs.<locals>.<listcomp>   s    """aqc"""r)   rh   rJ   rL   r:   r)   ro   r   )rO   r   r0   r   r^   r_   computeall)rc   rd   rz   rY   r{   rp   rS   results    `      r'   !test_text_blocks_to_pandas_kwargsr      s    ....u...F""6"""F&$(FFE*:$;S&IID#$**511!4u<F	vvvtV	D	DB
~----ZZ\\FNbj(--///////r)   c           	        t                      }|d                             d          d         dz   }g }t          |          D ]L}||         }|                    d          }|                    d t	          d|          D                        Mt          | |||                                i           }t          |                                	                    d          |	                    d          d	           |d
dg         }	t          | |||	                                dd
dgi          }t          |                                	                    d          |		                    d          d	           d S )Nr:   ro   r   c                8    g | ]}d                      |          S ro   )r,   )r%   bss     r'   r(   z6test_text_blocks_to_pandas_blocked.<locals>.<listcomp>   s"    HHH"uzz"~~HHHr)      TdropFcheck_dtyperJ   rL   rh   )
rT   r0   rO   appendr
   r   r{   r   r   reset_index)
rc   rd   expectedrp   rz   rA   re   linesrS   	expected2s
             r'   "test_text_blocks_to_pandas_blockedr      s   ||H#$**511!4u<FFE]] J J!HHHa0G0GHHHIIII	vvvx}}	K	KB


  d ++$''    &$(I		 0 09vtn2M
 
B 


  d ++4((     r)   zdd_read,pd_read,filesc                D   d |                                 D             }t          t                                                    }t	          |d          5   | d|          }t          |||          }t          ||d           d d d            d S # 1 swxY w Y   d S )Nc                0    i | ]\  }}|t           d z   |z   S r   )comment_headerr%   rJ   contents      r'   rC   z!test_skiprows.<locals>.<dictcomp>  s(    WWWgT>E)G3WWWr)   re   mode2014-01-*.csvskiprowsFr   )itemsr`   r   
splitlinesr   rZ   r   dd_readpd_readrd   skiprS   expected_dfs         r'   test_skiprowsr     s    
 XWWWWE~((**++D	5s	#	#	# 6 6W_t444%eWtDDD"ku55556 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6s   2BBBc                    d |                                 D             }t          |d          5   | dd          }t          ||d          }t          ||d           d d d            d S # 1 swxY w Y   d S )	Nc           	     Z    i | ](\  }}|t           d z   |                    d dd          z   )S )ro   s   # just some comment
r4   r   r@   r   s      r'   rC   z test_comment.<locals>.<dictcomp>  sQ        D' 	n

//%!91
=
=>  r)   re   r   r   #)commentFr   r   r   rZ   r   )r   r   rd   rS   r   s        r'   test_commentr     s    
  #[[]]	  E 
5s	#	#	# 6 6W_c222%eWcBBB"ku55556 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6s   2A//A36A3c                H   d |                                 D             }t          t                                                    }t	          |d          5   | d|d          }t          |||d          }t          ||d           d d d            d S # 1 swxY w Y   d S )	Nc                0    i | ]\  }}||d z   t           z   S r   )comment_footerr   s      r'   rC   z#test_skipfooter.<locals>.<dictcomp>*  s'    WWWgT7U?^3WWWr)   re   r   r   python)
skipfooterengineFr   )r   r`   r   r   r   rZ   r   r   s         r'   test_skipfooterr   %  s    
 XWWWWE~((**++D	5s	#	#	# 6 6W_hGGG%eWhWWW"ku55556 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6s   4BBBzdd_read,pd_read,files,unitsc                   fd|                                 D             }g d}t          |d          5   | d|          }t          |||          }t          ||d           d d d            d S # 1 swxY w Y   d S )	Nc           	     b    i | ]+\  }}|t           d z   |                    d d z   d          z   ,S )ro   r4   r   )r%   rJ   r   unitss      r'   rC   z)test_skiprows_as_list.<locals>.<dictcomp>:  sM       D' 	~%uu}a(P(PP  r)   )r   r4   r   r\      re   r   r   r   Fr   r   )r   r   rd   r   r   rS   r   s      `   r'   test_skiprows_as_listr   2  s       "[[]]  E ??D	5s	#	#	# 6 6W_t444%eWtDDD"ku55556 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6s   2A66A:=A:s   aa,bb
1,1.0
2,2.0s   10,20
30,40s   aa	bb
1	1.0
2	2.0s   10	20
30	40zreader,blocksc                4    | t          |d         d                   d          |d         d                             d          d         dz   }t          | ||i           }t          j        |d          }t          fd|D                       sJ d S )Nr   rp   ro   sync	schedulerc              3     K   | ]8}|j                                         j                                         k    V  9d S N)dtypesto_dict)r%   rS   r{   s     r'   	<genexpr>z&test_enforce_dtypes.<locals>.<genexpr>X  sD      JJry  ""dk&9&9&;&;;JJJJJJr)   )r   r0   r   ru   r   r   )rc   rz   rp   dfsr{   s       @r'   test_enforce_dtypesr   P  s     6'&)A,''222DAYq\&&q)E1F
b
A
AC
,sf
-
-
-CJJJJcJJJJJJJJJJr)   c           	        |d         |d         d                              dd          |d         d         gg} | t          |d         d                   d          }|d         d                             d          d         dz   }t          j        t
                    5  t          | |||i d          }t          j        |d	d
i d d d            d S # 1 swxY w Y   d S )Nr   r4      a   Ar   ro   Tenforcer   r   )	r@   r   r0   pytestraises
ValueErrorr   ru   r   )rc   rz   r{   rp   r   s        r'   test_enforce_columnsr   [  s#   
 Qi&)A,..tT::F1IaLIJF6'&)A,''222DAYq\&&q)E1F	z	"	" - -#FFFD"dSSSc,V,,,- - - - - - - - - - - - - - - - - -s   "'CCCzdd_read,pd_read,text,sepz\s+c                J   t          |          5 } | |dt          j        |          }t          |j                  ddgk    sJ |                    d                              d          }t          | |||	                     d d d            d S # 1 swxY w Y   d S )
N   )	blocksizelineterminatorseprJ   rK   r   r   Tr   r   )r   r5   linesepr^   r_   r   r   r   )r   r   textr   fnfr   s          r'   test_read_csvr   m  s     
$ 02GB"RZSIIIAI68"44444V,,88d8CC&''"#...///0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0s   A;BBBz-dataframe.convert-string requires pandas>=2.0reasonc                    t          j        dd           t          t                    5 } t                              |           }t          j                            ddi          5  t                              |           }d d d            n# 1 swxY w Y   |
                    ddi          }t          ||d	           d d d            d S # 1 swxY w Y   d S )
NpyarrowzRequires pyarrow stringsr   zdataframe.convert-stringTrJ   zstring[pyarrow]Fcheck_index)r   importorskipr   csv_textrF   rG   ru   configsetrw   rP   r   )r   rS   ddf
df_pyarrows       r'   #test_read_csv_convert_string_configr   ~  sC    	*DEEEE	(		 6r[[__[__8$?@@ 	" 	"++b//C	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	"YY(9:;;
*cu55556 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6s5   <C'BCB	CB	,CCCzdd_read,pd_read,text,skip   r4      c           	         ddg}t          |          5 } | |||          }t          | ||||                     d d d            d S # 1 swxY w Y   d S )NrJ   rK   r   names)r   r   r   r   r   r   r   r   actuals          r'   test_read_csv_large_skiprowsr     s     XE	$ C2d%888&''"t5AAABBBC C C C C C C C C C C C C C C C C Cs   +AAA   c           
         ddg}t          |          5 }t          j        t          d          5   | |d||                                          }t          | ||||                     d d d            n# 1 swxY w Y   t          j        t                    5  t          j        t                    5   | |d||           d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )	NrJ   rK   zsample=blocksizematch   )r   r   r   r   r   )r   r   warnsUserWarningr   r   r   r   r   s          r'   .test_read_csv_skiprows_only_in_first_partitionr     sV    XE	$ F2\+-?@@@ 	G 	GWR3UKKKSSUUFfggb4uEEEFFF	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	G \+&& 	F 	Fz** F Fb4uEEEEF F F F F F F F F F F F F F F	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	FF F F F F F F F F F F F F F F F F Fs|   D>A:.D:A>	>DA>	DC+8CC+CC+CC+D+C/	/D2C/	3DD
Dc                *   t                      }t          |d          5   | d          }t          ||d           d} | |          } |t          ||                             }t          ||d           d d d            d S # 1 swxY w Y   d S )Nre   r   r   Fr   r:   )rT   r   r   r   )r   r   rd   r   rS   r   r   s          r'   test_read_csv_filesr     s    
 ||H	5s	#	#	# 4 4W_%%"hE2222WR[[GGE"I..//	"iU33334 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4s   ABBBc                X   t          |d          5  t          |          d d         }t          |          } | |          }t          ||d           t	          j        t                    5   | g            d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nre   r   r   Fr   )r   rO   rT   r   r   r   r   )r   r   rd   subsetsolress         r'   test_read_csv_files_listr     s0   
 
5s	#	#	#  rr"  gfoo#s....]:&& 	 	GBKKK	 	 	 	 	 	 	 	 	 	 	 	 	 	 	                 s6   AB/B;BB	BB	BB#&B#zdd_read,filesc                    t          |d          5   | dddt          i          }|j                                                                        }d|v sJ d|vsJ d	|v sJ 	 d d d            d S # 1 swxY w Y   d S )
Nre   r   r   Tr6   include_path_column
convertersr:   r;   r<   )r   r7   r6   r   uniquer   rd   rS   	filenamess       r'   !test_read_csv_include_path_columnr    s     
5s	#	#	# 	- 	-W $/
 
 

 GOO%%,,..	9,,,,y00009,,,,,	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	-   AA33A7:A7c                    t          |d          5   | dddt          i          }|j                                                                        }d|v sJ d|vsJ d|v sJ 	 d d d            d S # 1 swxY w Y   d S )	Nre   r   r   filenamer   r:   r;   r<   )r   r7   r  r   r  r  s       r'   (test_read_csv_include_path_column_as_strr    s     
5s	#	#	# 	- 	-W *"N3
 
 

 K''))0022	9,,,,y00009,,,,,	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	- 	-r  c                    t          |d          5  t          j        t                    5   | dd           d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nre   r   r   rJ   r   )r   r   r   r   )r   rd   s     r'   5test_read_csv_include_path_column_with_duplicate_namer    s    
5s	#	#	# A A]:&& 	A 	AGO@@@@	A 	A 	A 	A 	A 	A 	A 	A 	A 	A 	A 	A 	A 	A 	AA A A A A A A A A A A A A A A A A As3   AAAA
	
AA
	AA"%A"c                X   t          |d          5   | dd          }|j        j        dk    sJ t          |j                  sJ  | dd          }|                                }|j        j        dk    sJ t          |j                  sJ 	 d d d            d S # 1 swxY w Y   d S )Nre   r   r   Tr
  category)r   r6   rl   r   r   r   rd   rS   r   r   s        r'   3test_read_csv_include_path_column_is_dtype_categoryr    s
    
5s	#	#	# 1 1W_$???w}
****#BG,,,,,go4@@@{ J....#FK0000001 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1s   A?BB#&B#c                v   t          |d          5   | ddd          }|j        dk    sJ |j        j        dk    sJ t	          |j                  sJ  | ddd          }|                                }|j        j        dk    sJ t	          |j                  sJ 	 d d d            d S # 1 swxY w Y   d S )	Nre   r   r   10BT)r   r   r\   r  )r   npartitionsr6   rl   r   r   r  s        r'   Ctest_read_csv_include_path_column_with_multiple_partitions_per_filer    s!    
5s	#	#	# 	1 	1W_4PPP~!!!!w}
****#BG,,,,,goDQQQ{ J....#FK000000	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1s   BB..B25B2c                 
   t          t                    5 } t                              | d                              d          }|                    d          }|j        j        dk    sJ t          t          j	        |j
        |                                d          }t          |          D ]}\  }}|t          |j                  dz
  k     r-|j        |j        |dz            k                                     sJ |dk    r*|j        |j        |         k                                    sJ ~t                               |                               d          }t#          ||           d d d            d S # 1 swxY w Y   d S )	N   r   rK   r   r   r   r4   r   )r   r   rw   rG   	set_indexr   indexrJ   r   rx   ru   __dask_keys__	enumerater`   	divisionsr   rF   r   )r   r   r   rz   iblockr   s          r'   test_read_csv_indexr    s   	(		 $rKKbK))33H==V,,| H,,,,)L!&!//"3"3v
 
 
 "&)) 	= 	=HAu3q{##a'''ak!a%&88==?????1uuq{1~5::<<<<<;;r??,,X66&(###$ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $s   EE88E<?E<c                 (   t          t                    5 } t                              | t	          d                    }|}t
                              | t	          d                    }t          ||           d d d            d S # 1 swxY w Y   d S )Nr   r   )r   r   rw   rG   rangerF   r   )r   r   r   r   s       r'   test_read_csv_skiprows_ranger!  +  s    	(		 $rKKU1XXK..;;rE!HH;55&(###	$ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $s   A%BBBc                    t          t                    5 } t                              | dddg          }|dg         }t                              | ddg          }|dg         }|                                j        |j        k                                    sJ |                                j        |j        k                                    sJ 	 d d d            d S # 1 swxY w Y   d S )Nr   HighLow)r   rh   rh   )r   
timeseriesrw   rG   rF   r   r|   r   )r   rS   	df_selectr   expected_selects        r'   test_usecolsr)  3  s+   	*		 L[[rFE?[CCxL	;;rFE?;;;"F8,

#x6;;=====!!##*o.DDIIKKKKKKL L L L L L L L L L L L L L L L L Ls   B<CC#&C#c                 H   t          t                    5 } t                              | d          }t                              | d          }|j        |j        k    sJ t                              | d          }|j        dk    sJ 	 d d d            d S # 1 swxY w Y   d S )N30Br  3064MiBr4   )r   r&  rw   rG   r  r   are   cs       r'   test_string_blocksizer1  =  s    	*		 "KKeK,,KKdK++}----KKgK..}!!!!!" " " " " " " " " " " " " " " " " "s   A4BBBc                 (   t          d          } t          |           5 }t                              |dd          }d|j        v sJ |j                                                                        dk    sJ 	 d d d            d S # 1 swxY w Y   d S )Nzm
    name, amount
    Alice,100
    Bob,-200
    Charlie,300
    Dennis,400
    Edith,-500
    Frank,600
    Tr  )skipinitialspacer   rK   iX  )r2   r   rw   rG   r_   rK   maxr   r   r   rS   s      r'   test_skipinitialspacer6  G  s    	
 
D 
$ 02[[db[AA2:%%%%y}}&&((C/////	0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0s   ABBBc                     t          d          } t          |           5 }t                              |d          }|j                                        j        t          k    sJ 	 d d d            d S # 1 swxY w Y   d S )Nzp
    name,amount
    Alice,100.5
    Bob,-200.5
    Charlie,300
    Dennis,400
    Edith,-500
    Frank,600
    r   r  )r2   r   rw   rG   rK   r   rl   rk   r5  s      r'   test_consistent_dtypesr8  [  s    	
 
D 
$ 22[[r[**y  ""(E111112 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2s   AA33A7:A7c                 Z   t          d          } t          d          }t                      }t          | |d          5  t                              dd          }|j        j        |k    sJ |j                                        j        |k    sJ 	 d d d            d S # 1 swxY w Y   d S )Nz@
    name,amount
    Alice,100
    Bob,-200
    Charlie,300
    z8
    name,amount
    1,400
    2,-500
    Frank,600
    z	foo.1.csvz	foo.2.csv	foo.*.csv   r  )r2   r   r   rw   rG   rJ   rl   r   )text1text2string_dtyperS   s       r'   test_consistent_dtypes_2r@  m  s    	 E 	 E $%%L	U;;	<	< 7 7[[[33w},,,,w  &,666667 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7s   AB  B$'B$c                    t          d          } t          d          }t          | |d          5  t                              dddid          }|j        j        dk    sJ t          |j                  rJ |                                }|j        j        dk    sJ t          |j        j	        j
                  g d	k    sJ 	 d d d            d S # 1 swxY w Y   d S )
NzJ
    fruit,count
    apple,10
    apple,25
    pear,100
    orange,15
    zO
    fruit,count
    apple,200
    banana,300
    orange,400
    banana,10
    r:  r;  fruitr  r<  )rl   r   )applebananaorangepear)r2   r   rw   rG   rB  rl   r   r   rO   cat
categories)r=  r>  rS   r   s       r'   test_categorical_dtypesrI    sO   	 E 	 E 
U;;	<	< Y Y[[Wj,AR[PPx~++++'11111jjlly*,,,,cim.//3X3X3XXXXXXY Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Ys   BCCCc                 
   t          d          } t          d          }t          j        j                            g dd          }t          | |d          5  t                              dddd	
          }|j        j	        j
        du sJ |j        j	        j
        du sJ t                              t                              g d|j                  t                              g d|j                  d	g d          }t          ||           t                              d|dd	
          }|j        j	        j
        du sJ |j        j	        j
        du sJ t!          j        |j        j	        j        |j                   |j        j	        j        du sJ t          ||           t          j        j                            g dd          }t                              d|dd	
          }|d         j	                                        |d<   |j        j	        j
        du sJ |j        j	        j
        du sJ |j        j	        j        du sJ t          ||           t                              dt          j        j                            d          
          }|j        j	        j
        du sJ t                              dd
          }|j        j	        j
        du sJ 	 d d d            d S # 1 swxY w Y   d S )Nz%
    A,B
    a,a
    b,b
    a,a
    z%
    A,B
    a,a
    b,b
    c,c
    )r/  re   r0  F)orderedr:  r;  r  )ABrl   )r/  re   r/  r/  re   r0  )rH  )r   r4   r   r   r4   r   r  TrL  )r2   rF   apitypesCategoricalDtyper   rw   rG   rL  rG  knownrM  rx   CategoricalrH  r   r   assert_index_equalrK  
as_ordered)r=  r>  rl   r   r   s        r'   test_categorical_knownrW    sd   	 E 	 E FL))///5)IIE	U;;	<	< *+ *+[jz0R0RSSx|!U****x|!U****<<^^222u?O $   ^^222u?O $  	  %$$   

 

 	&(### [e*0M0MNNx|!T))))x|!U****
fhl5u7GHHHx|#u,,,,&(### --ooot-LL[e*0M0MNN )4466x|!T))))x|!U****x|#t++++&(### rv|<<U<KK  
 
 x|!U****[
;;x|!U*****U*+ *+ *+ *+ *+ *+ *+ *+ *+ *+ *+ *+ *+ *+ *+ *+ *+ *+s   JK88K<?K<compressioninfergzipc                   t                      5 }t          j        t          j                            |d          d          }|                    t                                                     |	                                 t          j        t          j                            |d          d          }|                    t                                                     |	                                 t          j        t                    5  t                              t          j                            |d          |           }d d d            n# 1 swxY w Y   t          |                                          t          t                              d                    dz
  dz  k    sJ 	 d d d            d S # 1 swxY w Y   d S )	Nza.csv.gzwbzb.csv.gzz*.csv.gzrX  r+   r4   r   )r    rZ  openr5   r6   r,   writer   encodecloser   r   r   rw   rG   r`   r   r0   )rX  tdirr   rS   s       r'   test_compression_multiple_filesrc    s    
 HTIbgll444d;;	!!"""				Ibgll444d;;	!!"""				\+&& 	V 	VRW\\$
;;UUB	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 2::<<  S)=)=%>%>%Ba$GGGGGGH H H H H H H H H H H H H H H H H Hs7   D G;E
GE	GE	AGGGc                    t          d          5 } t                              | d          }t          |                                          dk    sJ t          |j                  ddgk    sJ 	 d d d            d S # 1 swxY w Y   d S )Nza,br   r   r/  re   )r   rw   rG   r`   r   r^   r_   r   rS   s     r'   test_empty_csv_filerf    s    	% .B[[A[&&2::<<  A%%%%BJC:-----. . . . . . . . . . . . . . . . . .s   A A>>BBc                     t          t          d          5 } t                              | d          }t	          |j                  g dk    sJ 	 d d d            d S # 1 swxY w Y   d S )Nre   r   FsamplerI   )r   rH   rw   rG   r^   r_   re  s     r'   test_read_csv_no_samplerj    s    	93	'	'	' <2[[E[**BJ#;#;#;;;;;;< < < < < < < < < < < < < < < < < <s   9AA"%A"c                     t          t          d          5  t                              dd          } t                              dd          }| j        |j        k    sJ 	 d d d            d S # 1 swxY w Y   d S )Nre   r   r   Tr   F)r   rH   rw   rG   _namer/  re   s     r'   "test_read_csv_sensitive_to_enforcern    s    	93	'	'	' " "KKK66KKK77w!'!!!!!" " " " " " " " " " " " " " " " " "s   AA00A47A4r   
   r&   c                   | r | t           vrt          j        d| z             t                      }ddddd                    | d          | r t          t           |          t                    nt          }fd|                                D             }t          |d	
          5  | rS|rQt          j	        t                    5  t                              dz   |          }d d d            n# 1 swxY w Y   nt                              dz   |          }t          |                    d                              d          |                    d          d           d d d            d S # 1 swxY w Y   d S )Nz%compress function not provided for %sz.gzz.bz2z.zipz.xz)rZ  bz2zipxz c                "    i | ]\  }}|z   |S r$   r$   )r%   rA   rB   suffixs      r'   rC   z-test_read_csv_compression.<locals>.<dictcomp>  s#    >>>tq!QZ>>>r)   re   r   r   r  r   r   Tr   Fr   )r   r   r   rT   getr   rH   r   r   r   r   rw   rG   r   r   r   )r&   r   r   files2renamed_filesrS   rv  s         @r'   test_read_csv_compressionrz  	  s     Cs("";cABBB||HF6GGKKCQSTTF14CVHSM9---)F>>>>v||~~>>>M	=s	+	+	+ 
 
  	L9 	Lk** P P[[6!9Y[OOP P P P P P P P P P P P P P P _v5KKBJJJ((44$4??  d ++	
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
s7   'E7 C1%E71C5	5E78C5	9A1E77E;>E;c                 ^   t          t          d         t                    } t          | d          5  t	          j        t                    5 }t                              dd          }|j	        dk    sJ 	 d d d            n# 1 swxY w Y   t          |          dk    sJ t          |d         j                  }d|v sJ d	|v sJ t          j        d
          5 }t                              ddd           }d d d            n# 1 swxY w Y   |rJ t	          j        t                     5  t	          j        t                    5  t                              dd          }d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )NrZ  re   r   r   r]  r\   r4   r   zblocksize=NoneT)record)rX  r   foo)r   r   rH   r   r   r   r   rw   rG   r  r`   r.   messagewarningscatch_warningsr   NotImplementedError)rx  wrS   msgr|  s        r'   test_warn_non_seekable_filesr  "  s   HV$i00F	6	$	$	$ E E\+&& 	'!_&AAB>Q&&&&&	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 1vv{{{{!A$,}}}}3&&&&$D111 	RV_&DQQB	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R].// 	E 	Ek** E E[[e[DDE E E E E E E E E E E E E E E	E 	E 	E 	E 	E 	E 	E 	E 	E 	E 	E 	E 	E 	E 	EE E E E E E E E E E E E E E E E E Es   F"*B7F"B	F"
B	AF"D<F"D	F"D	 F"0F

E3'F
3E77F
:E7;F
>F"
F	F"F	F""F&)F&c                 Z   d} t          |           5 }t                              |dd          }|j                                                                        dk    sJ |j                                                                        dk    sJ 	 d d d            d S # 1 swxY w Y   d S )Nz!a,b
1,2
2,3
3,4
4,5
5,6
6,7r   z
)r   r         )r   rw   rG   re   ra   r   r/  r5  s      r'   test_windows_line_terminatorr  8  s    :D	$ =2[[q[@@txxzz!!##'<<<<<txxzz!!##'<<<<<<= = = = = = = = = = = = = = = = = =s   B B  B$'B$rp   )r4   r   r\   c                    d}t          d|i          5  t                              d| d          }t                              d|           }t	          ||d           d d d            d S # 1 swxY w Y   d S )Nzid0,name0,x0,y0
id,name,x,y
1034,Victor,-0.25,0.84
998,Xavier,-0.48,-0.13
999,Zelda,0.00,0.47
980,Alice,0.67,-0.98
989,Zelda,-0.04,0.03
ztest_header_int.csv@   )rp   r   r   Fr   )r   rw   rG   rF   r   )rp   r   rS   r   s       r'   test_header_intr  @  s    	! 	 
)40	1	1 3 3[[.v[LL;;4V;DD"hE22223 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3s   AA--A14A1c                 D   t          dddd          5  t                              dd           } t                              ddgd	d
gd          }t          |                                                     d          |           d d d            d S # 1 swxY w Y   d S )Nz1,2rt  z3,4)z
.tmp.1.csvz
.tmp.2.csvz
.tmp.3.csvz
.tmp.*.csvr   r4   r\   r      )r   r4   Tr   )r   rw   rG   rF   rx   r   r   r   )rS   r   s     r'   test_header_Noner  Q  s    	%rOO	P	P A A[[d[33<<QF1v 6 677"**,,***55x@@@A A A A A A A A A A A A A A A A A As   A4BBBc                     t          t          dd          t                    sJ t          dd          dk    sJ t          dd          dk    sJ d S )Ni     r\   d   i  r      )rv   r   rQ   r$   r)   r'   test_auto_blocksizer  X  s^    nT2..44444$""c))))$""c))))))r)   c                    t          j        d          } G d d          fd}|                     |d|           t                      sJ dS )zp
    psutil returns a total memory of `None` on some systems
    see https://github.com/dask/dask/pull/7601
    psutilc                      e Zd ZdZdS )*test__infer_block_size.<locals>.MockOutputN)__name__
__module____qualname__totalr$   r)   r'   
MockOutputr  e  s        r)   r  c                      S r   r$   )r  s   r'   mock_virtual_memoryz3test__infer_block_size.<locals>.mock_virtual_memoryh  s	    r)   virtual_memoryN)r   r   setattrr   )monkeypatchr  r  r  s      @r'   test__infer_block_sizer  ^  s    
  **F             02EFFFr)   c                 ~    t          dd          } | t          d          k    sJ t          | t                    sJ d S )Nl    J)r\   g    A)r   rQ   rv   r  s    r'   test_auto_blocksize_max64mbr  o  sD    }a00ID		!!!!i%%%%%%%r)   c                
   t          j        d          }|                                j        }|                                }t          j        t                    }|                     t          j
        j        j        d|           t          ||          }t          t          d          5  t                               d           |j        sJ |j        d         d         |k    sJ 	 d d d            d S # 1 swxY w Y   d S )	Nr  )wrapsr   re   r   r:   r4   r   )r   r   r  r  	cpu_countr   Mockr   r  ru   	dataframeiocsvr   r   rH   rw   rG   called	call_args)r  r  total_memoryr  mock_read_bytesexpected_block_sizes         r'   test_auto_blocksize_csvr  u  sH    **F((**0L  ""Iij111O)-|_MMM(yAA	93	'	'	' P P
$%%%%%%%(+K8<OOOOOOP P P P P P P P P P P P P P P P P Ps   -=C88C<?C<c                    ddd} t          |           5  t                              dd           t                              dd          }|j        d	k                                    sJ 	 d d d            d S # 1 swxY w Y   d S )
NzAa,b
0,'abcdefghijklmnopqrstuvwxyz'
1,'abcdefghijklmnopqrstuvwxyz'z.a,b
111111,-11111
222222,-22222
333333,-33333
).overflow1.csv.overflow2.csvr  4   rh  r  #   i8)r   rw   rG   r   r   )rd   rS   s     r'   test_head_partial_line_fixr    s     RN	 E 
5		 ) )
$R000
 [[)"[55	T!&&(((((() ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) )s   AA;;A?A?c                     d} 	 t                               |            J # t          $ r}| t          |          v sJ Y d }~d S d }~ww xY w)Nz.not.a.real.file.csv)rw   rG   OSErrorr.   )r   es     r'    test_read_csv_raises_on_no_filesr    s^    	B
B   SVV||||||||||s     
AAAc                 B   t          t                    5 } t                              |           }t                              |           }|j        |j        k    sJ t          |j                                        t                    t          |j                                        t                    k    sJ t          |j        t                    sJ t                              | ddg          }|j        |j        k    sJ 	 d d d            d S # 1 swxY w Y   d S )Nkeyr4   r   )r   	na_values)
r   r   rw   rG   rl  rO   ru   keysr.   rv   r.  s       r'   $test_read_csv_has_deterministic_namer    s,   	(		 "rKKOOKKOOw!'!!!!afkkmm---31O1O1OOOOO!'3'''''KKQ1#K66w!'!!!!!" " " " " " " " " " " " " " " " " "s   C1DDDc                    t          t          t          d          5  t                              d          } t                              d          }t	          | j                                        t                    t	          |j                                        t                    k    sJ 	 d d d            d S # 1 swxY w Y   d S )N)z
_foo.1.csvz
_foo.2.csvz
_foo.*.csvr  )r   r   rw   rG   rO   ru   r  r.   rm  s     r'   -test_multiple_read_csv_has_deterministic_namer    s    	((CC	D	D P PKK%%KK%%afkkmm---31O1O1OOOOOO	P P P P P P P P P P P P P P P P P Ps   BB>>CCc                     t          t                    5 } t                              | d          }t                              | d          }|j        |j        k    sJ 	 d d d            d S # 1 swxY w Y   d S )N10kBr  20kB)r   r   rw   rG   rl  )r   r/  re   s      r'   4test_read_csv_has_different_names_based_on_blocksizer    s    	(		 "rKKfK--KKfK--w!'!!!!!" " " " " " " " " " " " " " " " " "s   AA..A25A2c                     t          d          5 } t                              | d           }t          |j                  ddgk    sJ 	 d d d            d S # 1 swxY w Y   d S )Nzalice,1
bob,2r   r   r4   )r   rw   rG   r^   r_   re  s     r'   test_csv_with_integer_namesr    s    	"	#	# *r[[D[))BJAq6)))))* * * * * * * * * * * * * * * * * *s   9AAAc                 d   d} t          d          D ]}| dz  } | dz  } d}t          |           5 }t                              |          }d}t	          j        t                    5 }t                              |ddg	                              d
           d d d            n# 1 swxY w Y   |	                    ||z             sJ t	          j        t                    5 }t                              |d                              d
           d d d            n# 1 swxY w Y   |	                    |          sJ d}t	          j        t                    5 }t                              |dddi                              d
           d d d            n# 1 swxY w Y   t          |j                  |k    sJ t	          j        t                    5 }t                              |ddgddi                              d
           d d d            n# 1 swxY w Y   t          |j                  ||z   k    sJ d}t	          j        t                    5 }t                              |ddgt          t          t          d                              d
           d d d            n# 1 swxY w Y   t          |j                  |k    sJ t                              |dt          t          t          d          }t          ||           d d d            d S # 1 swxY w Y   d S )Nz*numbers,names,more_numbers,integers,dates
  z1,,2,3,2017-10-31 00:00:00
z"1.5,bar,2.5,3,4998-01-01 00:00:00
a\  

-------------------------------------------------------------

The following columns also failed to properly parse as dates:

- dates

This is usually due to an invalid value in that column. To
diagnose and fix it's recommended to drop these columns from the
`parse_dates` keyword, and manually convert them to dates later
using `dd.to_datetime`.aU  Mismatched dtypes found in `pd.read_csv`/`pd.read_table`.

+--------------+---------+----------+
| Column       | Found   | Expected |
+--------------+---------+----------+
| more_numbers | float64 | int64    |
| names        | object  | float64  |
| numbers      | float64 | int64    |
+--------------+---------+----------+

- names
  ValueError(.*)

Usually this is due to dask's dtype inference failing, and
*may* be fixed by specifying dtypes manually by adding:

dtype={'more_numbers': 'float64',
       'names': 'object',
       'numbers': 'float64'}

to the call to `read_csv`/`read_table`.2   dates)ri  parse_datesr   r   rh  aa  Mismatched dtypes found in `pd.read_csv`/`pd.read_table`.

+--------------+---------+----------+
| Column       | Found   | Expected |
+--------------+---------+----------+
| more_numbers | float64 | int64    |
| numbers      | float64 | int64    |
+--------------+---------+----------+

Usually this is due to dask's dtype inference failing, and
*may* be fixed by specifying dtypes manually by adding:

dtype={'more_numbers': 'float64',
       'numbers': 'float64'}

to the call to `read_csv`/`read_table`.

Alternatively, provide `assume_missing=True` to interpret
all unspecified integer columns as floats.r   O)ri  rl   )ri  r  rl   aQ  Mismatched dtypes found in `pd.read_csv`/`pd.read_table`.

The following columns failed to properly parse as dates:

- dates

This is usually due to an invalid value in that column. To
diagnose and fix it's recommended to drop these columns from the
`parse_dates` keyword, and manually convert them to dates later
using `dd.to_datetime`.)more_numbersr   numbers)r   r   rF   rG   r   r   r   rw   r   r   r.   valuerk   objectr   )r   _date_msgr   r   r  r  r   s           r'   test_late_dtypesr    s   8D4[[ / /..11D	"  
$ ^2kk"oo6 	0 ]:&& 	X!KK2G9K==EEPVEWWW	X 	X 	X 	X 	X 	X 	X 	X 	X 	X 	X 	X 	X 	X 	XwwsX~&&&&&]:&& 	A!KK2K&&...@@@	A 	A 	A 	A 	A 	A 	A 	A 	A 	A 	A 	A 	A 	A 	Awws||9 	, ]:&& 	W!KK2gs^K<<DDvDVVV	W 	W 	W 	W 	W 	W 	W 	W 	W 	W 	W 	W 	W 	W 	W17||s""""]:&& 	(!KK2G9WcN   gg'''	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 17||sX~----	& 	 ]:&& 	(!KK$I',v%PP	   
 gg'''	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 17||s"""" kk#(6eLL  
 

 	#s}^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^s   6L%&3B%L%%B)	)L%,B)	-6L%#1D L% D$	$L%'D$	(5L%4FL%F!	!L%$F!	%6L%6HL%H!	!L%$H!	%;L% AJ3'L%3J7	7L%:J7	;AL%%L),L)c                 d   d} t          d          D ]}| dz  } | dz  } t          |           5 }t                              |          }t                              |dd          }t          ||                    dt          i                     t                              |dddd	i
          }t          ||           t                              |ddd 
          }t          ||                    dt          i                     d d d            n# 1 swxY w Y   d} t          d          D ]}| dz  } | dz  } t          |           5 }t                              |          }t                              |dd	d          }|j        j	        d	k    sJ 	 d d d            d S # 1 swxY w Y   d S )Nz$numbers,names,more_numbers,integers
r  z
1,foo,2,3
z1.5,bar,2.5,3
r  T)ri  assume_missingintegersint64)ri  r  rl   znumbers,integers
z1,2
z1.5,2
r   )ri  rl   r  )
r   r   rF   rG   rw   r   rP   rk   r  rl   )r   r  r   r   r   rS   s         r'   test_assume_missingr  5  s1   2D4[[  D	$ 82kk"oo kk"Rk==#szz:u"566777 kkr$z76K  
 
 	#s kk"RDkII#szz:u"5667778 8 8 8 8 8 8 8 8 8 8 8 8 8 8"  D4[[  ID	$ +2kk"oo [[Bgd[KKz7*****+ + + + + + + + + + + + + + + + + +s%   CDDDAF%%F),F)c                    t          t                    5 } 	 t                              | dd           J # t          $ r}dt          |          v sJ Y d }~nd }~ww xY wt                              | d          }t                              | dd          }t          ||d           d d d            d S # 1 swxY w Y   d S )Nr   rJ   )r   	index_colFr  )r  r   )r   r   rw   rG   r   r.   rF   r   )r   r  rS   r   s       r'   test_index_colr  X  s   	(		 	.r	)KKbFK;;;L 	) 	) 	)#a&&(((((((((	) [[u[--kk"ek<<"cu----	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	.s3   B86
A AB8AAB88B<?B<c                    t          t                    5 } t                              | ddddgdg          }t                              | dddgdgd                              d          }t          ||           t                              | dddgdg                              d          }t          ||           d d d            d S # 1 swxY w Y   d S )Nr   r  Dater  rp   rh   r  i rp   rh   r  r   )rp   rh   r  r   r&  rF   rG   rw   r  r   r   rS   r   s      r'   0test_read_csv_with_datetime_index_partitions_oner  e  s-   	*		 [[!A1vF8  
 
 kkq1a&vh(  
 

)F

 	 	"c kk"QAVHkMMWW
 
 	"c                 s   B,CCCc                 .   t          t                    5 } t                              | ddddgdg          }t                              | dddgdgd                              d          }t          ||           d d d            d S # 1 swxY w Y   d S )Nr   r  r  r  i  r  r  r  s      r'   .test_read_csv_with_datetime_index_partitions_nr  w  s    	*		 [[!A1vF8  
 
 kkq1a&vh#  
 

)F

 	 	"c                 s   A(B

BBz(https://github.com/dask/dask/issues/5787encodingzutf-16)marksz	utf-16-lez	utf-16-bec                8   t                               t          dd                    }|dz  }|dz  }|dz  }t                               ||||d          }t	          d          5 }|                    || d	           t                               || 
          }t                              || d          }|                                }t          t          |j
                            |_
        t          ||           d d d            d S # 1 swxY w Y   d S )Nr   r  r   gffffff
@gZӼ?r/  re   r0  d.csvF)r  r  )r  r  )r  r   )rF   Seriesr   rx   r!   to_csvrG   rw   r   r`   r  r   )	r  arbrcrdrtest_dfr   r/  r  s	            r'   test_encoding_gh601r    s7    
5C==	!	!B	aB	cB	fBll"2B??@@G	 BrHE:::KKXK..KKXK>>IIKKAG%%!Q                 s   &BDDDc                    d                     dd          } t          |           5 }t                              |d          }t	          |t
                              |d                     t                              |d          }t	          |t
                              |d                     d d d            d S # 1 swxY w Y   d S )Nza b c-d
1 2 3
4 5 6 r9   r   )	delimiter)r@   r   rw   rG   r   rF   r5  s      r'   test_read_csv_header_issue_823r    s    &..sD99D	$ 72[[[&&"bkk"$k//000[[t[,,"bkk"k556667 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7s   BC  CCc                     t          t                    5 } t                              | d           }t	          |t
                              | d                      d d d            d S # 1 swxY w Y   d S )Nr%  )r   r   rw   rG   r   rF   re  s     r'   test_none_usecolsr    s    	(		 5r[[T[**"bkk"dk334445 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5s   AA))A-0A-c                 ~   t          d          } t          |           5 }t                              |ddgg          }t                              |ddgg          }|j        |j        k                                    sJ t          |          t          |          k    sJ 	 d d d            d S # 1 swxY w Y   d S )NaX  
    ID,date,time
    10,2003-11-04,180036
    11,2003-11-05,125640
    12,2003-11-01,2519
    13,2003-10-22,142559
    14,2003-10-24,163113
    15,2003-10-20,170133
    16,2003-11-11,160448
    17,2003-11-03,171759
    18,2003-11-07,190928
    19,2003-10-21,84623
    20,2003-10-25,192207
    21,2003-11-13,180156
    22,2003-11-15,131037
    datetime)r  r2   r   rw   rG   rF   r_   r   r`   )	pdmc_textr   r   rS   s       r'   test_parse_dates_multi_columnr    s   	 I& 
)		 #kk"FF+;*<k==[[66*:);[<<
ck)..000002ww#c(("""""# # # # # # # # # # # # # # # # # #s   BB22B69B6c                 v   t          d          } t          |           5 }t                              |dd          }t                              |dd          }|j        |j        k                                    sJ t          |          t          |          k    sJ 	 d d d            d S # 1 swxY w Y   d S )NzB
    name###amount
    alice###100
    bob###200
    charlie###300z###r   )r   r   r   )sep_textr   r   rS   s       r'   test_read_csv_sepr    s    	 H 
(		 #rkk"%k99[[x[88
ck)..000002ww#c(("""""# # # # # # # # # # # # # # # # # #s   BB..B25B2c            	         ddz  dz   } t          | d          5 }t                              |d dddd	gd
                              d           d d d            d S # 1 swxY w Y   d S )Ns   0,my
1,data
r  s	   2,foobarr\  r   r8   r+   r/  re   r   )rp   r   r   r   r   r   r   )r   rw   rG   r   datar   s     r'   test_read_csv_slash_rr	    s    t#m3D	$T	"	"	" $b
* 	 	
 	
 'F'
#
#
#$ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $s   7AA"%A"c                     d} t          | d          5 }t          t                              |t                    t
                              |t                               d d d            d S # 1 swxY w Y   d S )Ns   a,b
1,2
3,4
5,6r\  r   rN  )r   r   rF   rG   rk   rw   r  s     r'   test_read_csv_singleton_dtyper    s     D	$T	"	"	" Nb"++b+..Be0L0LMMMN N N N N N N N N N N N N N N N N Ns   AA00A47A4zarrow engine available from 1.4c                 $   t          j        d           t          d          } t          |           5 }t	          t
                              |d          t                              |d                     d d d            d S # 1 swxY w Y   d S )Nr   z
    a,b
    1,2
    )r   )r   r   r2   r   r   rF   rG   rw   )r  r   s     r'   test_read_csv_arrow_enginer    s    
	"""	 H 
(		 Xr"++b+33R[[I[5V5VWWWX X X X X X X X X X X X X X X X X Xs   ABB	B	c                    t                                           } t          |           d         }| |                             dd          | |<   t	          | d          5  t
                              dd dg d	          }t                              d
          }|j        |j        k    	                                sJ t          ||           d d d            d S # 1 swxY w Y   d S )N   names   Namere   r   r   r4   rI   )rp   r   r   r:   )rH   copyrO   r@   r   rw   rG   rF   r_   r   r   )rd   rA   r   rS   s       r'   test_robust_column_mismatchr    s"   NNEubAQx11E!H	5s	#	#	#  kkD1<T<T<T  
 
 [[)**
ck)..00000#s                 s   A/CC"Cc                    t                                           } t          |           d         }| |                             dd          | |<   t	          | d          5  t
                              d          }|j        g dk                                    sJ |	                                j        g dk                                    sJ 	 d d d            d S # 1 swxY w Y   d S )	Nr  r  s   addressre   r   r   rI   )rJ   rK   rL   address)
rH   r  rO   r@   r   rw   rG   r_   r   r   )rd   rA   r   s      r'   "test_different_columns_are_allowedr  
  s4   NNEubAQx44E!H	5s	#	#	# T Tkk/** 7777<<>>>>>%)L)L)LLQQSSSSSST T T T T T T T T T T T T T T T T Ts   A/CC #C c            	     ,   d} t          |           5 }d}t          j        t                    5  t                              ||           d d d            n# 1 swxY w Y   t          t                              ||d           t                              |d                      d d d            n# 1 swxY w Y   d}|| z   } t          |           5 }dt          |          z   }t          j        t                    5  t                              ||d           d d d            n# 1 swxY w Y   t          t                              ||d d	          t                              |d d
                     d d d            d S # 1 swxY w Y   d S )NzAAAAAA,BBBBB,CCCCC,DDDDD,EEEEE
1,2,3,4,5
6,7,8,9,10
11,12,13,14,15r  rh  )ri  rp   r   z# skip
# these
# lines
r\   )ri  r   )ri  rp   r   )rp   r   )	r   r   r   r   rw   rG   r   rF   r`   )r   r   ri  skiptexts       r'   !test_error_if_sample_is_too_smallr    sk   QD	$ 	
2]:&& 	+ 	+KK6K***	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	KK6$K77RPT9U9U	
 	
 	
	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 ,Hd?D	$ 

2c(mm#]:&& 	7 	7KK6AK666	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	KK6$KCCKK4!K44	
 	
 	


 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

sl   B3AB3A	B3A	AB33B7:B7,F	 D*F	*D.	.F	1D.	2A
F		FFc                     d} ddg}t          |           5 }t                              ||d          }t                              ||          }t	          ||d           d d d            d S # 1 swxY w Y   d S )	Nz?Alice,100
Bob,-200
Charlie,300
Dennis,400
Edith,-500
Frank,600
rJ   rK      )r   r   )r   Fr   )r   rw   rG   rF   r   )r   r   r   r   rS   s        r'   test_read_csv_names_not_noner  3  s    	 	 XE	$ .2kk"ERk88[[5[))"cu----. . . . . . . . . . . . . . . . . .s   AA//A36A3c            	     V   t                               g dg dd          } dD ]}t                              | |          }t	                      5 |                    d           t                              t          j        	                    d                    
                                                    d	          }t          ||            d d d            n# 1 swxY w Y   t	                      5 |                    dd
          }t          j
        |ddi}|t          fdt          |          D                       k    sJ t                              t          j        	                    d                    
                                                    d	          }t          ||            d d d            n# 1 swxY w Y   t	                      5 t          j        	                    d          }|                    |d          }|fdt          |          D             k    sJ t                              |          
                                                    d	          }t          ||            d d d            n# 1 swxY w Y   d S )Nr  r4   r   r\   r  xyr4   r   FrO  *Tr   )r  r   r   r   c              3  ^   K   | ]'}t           j                            | d           V  (dS )z.partNr5   r6   r,   r%   rW   dns     r'   r   ztest_to_csv.<locals>.<genexpr>V  sJ       " "23RA--" " " " " "r)   
data_*.csvc                X    g | ]&}t           j                            d | d          'S )data_r  r$  r%  s     r'   r(   ztest_to_csv.<locals>.<listcomp>_  s>       67R11  r)   )rF   rx   rw   from_pandasr    r  rG   r5   r6   r,   r   r   r   ru   tupler   )rS   r  r/  r   rpathsr   r&  s          @r'   test_to_csvr.  H  sK   	000|||DD	E	EB " "NN2{++XX 	"HHRuH%%%[[b#!6!677??AAMMSWMXXFfb!!!	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	"
 XX 	"5%88AL!6v66EE " " " "7<[7I7I" " "       [[b#!6!677??AAMMSWMXXFfb!!!	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" XX 	"b,//BHHRuH--E    ;@;M;M       [[__,,..:::EEFfb!!!	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	"#" "s9   BC##C'	*C'	<CGG	G	'B)JJ 	#J 	c                    t                               g dg dd          } t                              | d          }t	                      5 }t          j        t                    5  t          j	        
                    |d          }|                    |           d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   t                               g dg dd          }t                              |d          }t	                      5 }t          j	        
                    |d	          }|                    |d
           t                              |                                                              d          }t          ||           d d d            n# 1 swxY w Y   t                              | d          }t	                      5 }|                    |d
           t          j	        
                    |d	          }|                    |dd
           t                              |                                                              d          }t          ||            d d d            n# 1 swxY w Y   t                              |d          }t	                      5 }|                    |d
           t          j	        
                    |d	          }|                    |dd
           t                              |                                                              d          }t          ||           d d d            d S # 1 swxY w Y   d S )Nr  r  r  r   zdata_*_*.csv)r/  re   r0  r  r  r   ghr  jrA   lmrW   op)r4   r   r\   r  r   r]   r      	   ro     r   r      r  r  r  r'  FrO  Tr   r  )r   r  )rF   rx   rw   r*  r    r   r   r   r5   r6   r,   r  rG   r   r   r   )rS   r/  r&  r   df16r   s         r'   &test_to_csv_multiple_files_cornercasesr<  f  s   	000|||DD	E	EB
r1A	 R]:&& 	 	b.11BHHRLLL	 	 	 	 	 	 	 	 	 	 	 	 	 	 	              
 <<  $ IHH'	
 	
 D. 	tR  A	  RW\\"l++	5!!!R((**66D6AA&$	                              	r1A	 R	5!!!W\\"l++	#U+++R((**66D6AA&"               	tR  A	  R	5!!!W\\"l++	#U+++R((**66D6AA&$                                   so   B>%6B'B>'B+	+B>.B+	/B>>CCBF''F+.F+B JJJ;B M((M,/M,c                 r   t                               g dg dd          } dD ]}t                              | |          }t	                      5 }t
          j                            |d          }|                    |dd           t          	                    |          
                                                    d	          }t          ||            d d d            n# 1 swxY w Y   t	                      5 }t
          j                            |d          }|                    |ddd
          }t          j
        |d           t          	                    |          
                                                    d	          }t          ||            d d d            n# 1 swxY w Y   d S )Nr  r  r  r!  test.csvFT)r  single_filer   )r  r   r?  r   r   )rF   rx   rw   r*  r    r5   r6   r,   r  rG   r   r   r   ru   )rS   r  r/  r&  r   r   r,  s          r'   test_to_single_csvr@    s   	000|||DD	E	EB " "NN2{++XX 	"b*--BHHRu$H777[[__,,..:::EEFfb!!!		" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" XX 	"b*--B5%TJJALf----[[__,,..:::EEFfb!!!	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	"" "s&   B	C%%C)	,C)	>B F**F.	1F.	c                    t                               g dg dd          } t                              | d          }t	                      5 }t
          j                            |d          }t          j	        t          d          5  |                    |d d	d
           d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nr  r  r  r4   r>  z9name_function is not supported under the single file moder   c                    | S r   r$   )r  s    r'   <lambda>z7test_to_single_csv_with_name_function.<locals>.<lambda>  s     r)   FT)name_functionr  r?  rF   rx   rw   r*  r    r5   r6   r,   r   r   r   r  rS   r/  r&  r   s       r'   %test_to_single_csv_with_name_functionrG    sm   	000|||DD	E	EB
r1A	 SRW\\"j))]M
 
 
 	S 	S HHR{{%THRRR		S 	S 	S 	S 	S 	S 	S 	S 	S 	S 	S 	S 	S 	S 	SS S S S S S S S S S S S S S S S S Ss6   <CB."C.B2	2C5B2	6CC
C
c                    t                               g dg dd          } t                              | d          }t	                      5 }t
          j                            |d          }t          j	        t          d          5  |                    |ddd	
           d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nr  r  r  r4   r>  zDheader_first_partition_only cannot be False in the single file mode.r   FT)r  header_first_partition_onlyr?  rE  rF  s       r'   3test_to_single_csv_with_header_first_partition_onlyrJ    sR   	000|||DD	E	EB
r1A	 RW\\"j))]X
 
 
 	 	 HH%UPT    		 	 	 	 	 	 	 	 	 	 	 	 	 	 	                 s6   <CB-!C-B1	1C4B1	5CC	C	c                    t                               g dg dd          } t                              | d          }t	                      5 }t
          j                            |d          }|                    |ddd	
           t          	                    t
          j                            |d                    
                                }d d d            n# 1 swxY w Y   t          || d           d S )Nr  r  r  r   r  r>  Fr  Tr  r   r?  r"  r   )rF   rx   rw   r*  r    r5   r6   r,   r  rG   r   r   )df0rS   	directorycsv_pathr   s        r'   /test_to_csv_with_single_file_and_exclusive_moderQ    s    
,,111EE
F
FC		+	+B	 EY7<<	:66
		(%ct	DDDRW\\)S99::BBDDE E E E E E E E E E E E E E E fcu------s   BCC #C c                 b   t                               g dg dd          } t                              | d          }t	                      5 }t
          j                            t          |          d          }|	                    |ddd	
           t
          j        
                    |          sJ t          j        t                    5  |	                    |ddd	
           d d d            n# 1 swxY w Y   |	                    |ddd	
           d d d            d S # 1 swxY w Y   d S )Nr  r  r  r   rL  r>  Fr  TrM  r  )rF   rx   rw   r*  r    r5   r6   r,   r.   r  existsr   r   FileExistsError)rN  rS   rO  rP  s       r'   2test_to_csv_single_file_exlusive_mode_no_overwriterU    s   
,,111EE
F
FC		+	+B	 EY7<<I
;;
		(%ct	DDDw~~h''''']?++ 	I 	IIIhe#4IHHH	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 			(%ct	DDDE E E E E E E E E E E E E E E E E Es7   BD$C3'D$3C7	7D$:C7	;D$$D(+D(c                    t                               g dg dd          } dD ]}t                              | |          }t	                      5 }t
          j                            |d          }|                    |ddd	           t           	                    |d
          
                    d          }t          ||            d d d            n# 1 swxY w Y   d S )Nr  r  r  r!  ztest.csv.gzFrZ  T)r  rX  r?  r]  r   )rF   rx   rw   r*  r    r5   r6   r,   r  rG   r   r   )rS   r  r/  r&  r   r   s         r'   test_to_single_csv_gziprW    s   	000|||DD	E	EB " "NN2{++XX 	"b-00BHHRu&dHKKK[[[88DD$DOOFfb!!!		" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	"" "s   A:CC	C	z#to_csv does not support compressionc                 p   t                               g dg ddg d          } dD ]}t                              | |          }t	          d          5 }|                    |d	           t                               |d
d          }t          j        ||            d d d            n# 1 swxY w Y   d S )Nr  r  r  g      ?g       @g      @g      @rO  r!  r  rZ  r]  r   )r  rX  )	rF   rx   rw   r*  r!   r  rG   r   assert_frame_equal)rS   r  r/  r   r   s        r'   test_to_csv_gzipr[    s   	"""66>R>R>R 
 
 
B  . .NN2{++U^^ 	.rHHRVH,,,[[qf[EEF!&"---	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	.. .s   A
B**B.	1B.	z2023.9.1z)https://github.com/dask/dask/issues/10515c                    t                               g dg ddg d          } t                              | d          }t	                      5 }t
          j                            t          |          d          }|	                    |           dt          j
        |          v sJ t          j
        |          sJ t                              t
          j                            |d	                                                    }d d d            n# 1 swxY w Y   |j        j        | j        j        k                                    sJ d S )
Nr  r  r  rY  rO  r   rL  createmer"  )rF   rx   rw   r*  r    r5   r6   r,   r.   r  listdirrG   r   r  r|   r   )rN  rS   dirdir0r   s        r'   test_to_csv_nodirra    sp    ,,"""66>R>R>R   C 
	+	+B	 @Sw||CHHj11
		$RZ__,,,,z$RW\\$4455==??@ @ @ @ @ @ @ @ @ @ @ @ @ @ @ HOsu|+002222222s   B;DDDc                 8   t                               g dg ddg d          } t                              | d          }t	                      5 }t          |          }|                    |           t          j        |          sJ t          	                    t          j
                            |d                                                    }d d d            n# 1 swxY w Y   |j        j        | j        j        k                                    sJ d S )	Nr  r  r  rY  rO  r   rL  r"  )rF   rx   rw   r*  r    r.   r  r5   r^  rG   r6   r,   r   r  r|   r   rN  rS   r_  r   s       r'   test_to_csv_simplerd    s3   
,,"""66>R>R>R   C 
	+	+B	 ?S#hh
		#z#RW\\#s3344<<>>	? ? ? ? ? ? ? ? ? ? ? ? ? ? ?
 HOsu|+002222222s   BC!!C%(C%c                    t                               ddgddgd          } t                               ddgdd	gd          }t                              |d
          }t	                      5 }t
          j                            |d          }|                     |d           |                    |dddd           t          	                    t
          j                            |d                    
                                }d d d            n# 1 swxY w Y   t                               | |g          }t          ||d          sJ d S )Nr/  re   r4   r   r  r0  r  r\   r  rL  r>  FrO  T)r   rp   r  r?  r"  r   )rF   rx   rw   r*  r    r5   r6   r,   r  rG   r   rN   r   )rN  df1rS   r_  rP  r   r   s          r'   ,test_to_csv_with_single_file_and_append_moderg    s   
,,Cj1v&& C ,,Cj1v&& C 
	+	+B	 ?S7<<Z00

 	 	
 	
 	
 			 	 	
 	
 	
 RW\\#s3344<<>>? ? ? ? ? ? ? ? ? ? ? ? ? ? ? yy#s$$HVX599999999s   -BDDDc                    t                               g dg d          } t                              | d          }t	                      5 }t          |          }|                    |d           t          j        |          sJ t          	                    t          j
                            |d          d d	g
                                          }d d d            n# 1 swxY w Y   |j        | k                                    sJ d S )Nr  rY  rO  r   rL  Fr   r"  r  )rp   r   )rF   r  rw   r*  r    r.   r  r5   r^  rG   r6   r,   r   r  r   rc  s       r'   test_to_csv_seriesri  9  s6   
))(((0D0D0D)
E
EC		+	+B	 YS#hh
		#e	$$$z#RW\\#s33DNNVVXX	Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y
 HO  """""""s   BC""C&)C&c                    ddl m dgfd} t                              g dg dd          }t                              |d	          }t                      5 }|                    |dd
| i           d         sJ t                              t          j
                            |d                    }t          ||d           d d d            d S # 1 swxY w Y   d S )Nr   rw  Fc                     dd<    | i |S )NTr   r$   )argsrY   flagmp_gets     r'   my_getz$test_to_csv_with_get.<locals>.my_getI  s!    Qvt&v&&&r)   r  r  r  r   rL  r   r  compute_kwargsr"  r   )dask.multiprocessingrw  rF   rx   rw   r*  r    r  rG   r5   r6   r,   r   )rp  rS   r   r&  r   rn  ro  s        @@r'   test_to_csv_with_getrt  D  sA   2222227D' ' ' ' ' ' 
000|||DD	E	EB
...
+
+C	 1R

2UK3H
IIIAwRW\\"c2233&"%0000	1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1s   A/CC Cc                 x   ddl m t                              g dg dd          } t                              | d          }fd}t                      5 }t          j        t                    5  |
                    |d	|
           d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nr   rk  r  r  r  r   rL  c                      | i |S r   r$   rm  rY   ro  s     r'   rp  z:test_to_csv_warns_using_scheduler_argument.<locals>.my_get]      vt&v&&&r)   F)r  r   )rs  rw  rF   rx   rw   r*  r    r   r   FutureWarningr  rS   r   rp  r&  ro  s       @r'   *test_to_csv_warns_using_scheduler_argumentr{  W  sQ   222222	000|||DD	E	EB
...
+
+C' ' ' ' ' 
 :R\-(( 	: 	:JJr&J999	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	:: : : : : : : : : : : : : : : : : :s6   B/2BB/B	B/B	B//B36B3c            	        ddl m t                              g dg dd          } t                              | d          }fd}t                      5 }t          j        t                    ot          j
        t                    5  |                    |d	|d
|i           d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nr   rk  r  r  r  r   rL  c                      | i |S r   r$   rw  s     r'   rp  z@test_to_csv_errors_using_multiple_scheduler_args.<locals>.my_getk  rx  r)   Fr   )r  r   rr  )rs  rw  rF   rx   rw   r*  r    r   r   r   r   ry  r  rz  s       @r'   0test_to_csv_errors_using_multiple_scheduler_argsr~  e  sr   222222	000|||DD	E	EB
...
+
+C' ' ' ' ' 
 R]:&&F6<+F+F 	 	JJ%6;PVBW    	 	 	 	 	 	 	 	 	 	 	 	 	 	 	                 s6   3CB3'C3B7	7C:B7	;CCCc                    ddl m fd} t                              g dg dd          }t                              |d          }t                      5 }|                    |d	| d
d           d d d            d S # 1 swxY w Y   d S )Nr   rk  c                 0    |d         dk    sJ  | i |S )Ntest_kwargs_passedfoobarr$   rw  s     r'   rp  zBtest_to_csv_keeps_all_non_scheduler_compute_kwargs.<locals>.my_getx  s/    *+x7777vt&v&&&r)   r  r  r  r   rL  Fr  )r   r  rq  )rs  rw  rF   rx   rw   r*  r    r  )rp  rS   r   r&  ro  s       @r'   2test_to_csv_keeps_all_non_scheduler_compute_kwargsr  u  s    222222' ' ' ' ' 
000|||DD	E	EB
...
+
+C	 
R

)/xPP 	 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
s   BBBc                 ~   t                               dt          d          i          } t                              | d          }|                    d          }|d                             d          sJ |d                             d	          sJ t          j        d           t          j        d	           d S )
NrL  ro  r   rL  foo*.csvr   foo0.csvr4   zfoo1.csv)	rF   rx   r   rw   r*  r  endswithr5   remove)rS   r   r-  s      r'   test_to_csv_pathsr    s    	sE"II&	'	'B
...
+
+CJJz""E8Z(((((8Z(((((IjIjr)   zheader, expected)Frt  )Tx,y
c                   t                               g g d          }t                              |d          }t	                      5 }|                    t          j                            |d          d|            t          j        	                    t          j                            |d                    rJ t          j                            |d          }t          |          5 }|                                }||k    sJ 	 d d d            n# 1 swxY w Y   t          j        |           d d d            d S # 1 swxY w Y   d S )	Nr  r4   rL  z	fooe*.csvFr  rp   z	fooe1.csvz	fooe0.csv)rF   rx   rw   r*  r    r  r5   r6   r,   rS  r^  readliner  )rp   r   dfeddfer&  r  fplines           r'   "test_to_csv_header_empty_dataframer    s   
,,Rb))
*
*C>>#1>--D	 RBGLL[11vNNN7>>"',,r;"?"?@@@@@7<<K00(^^ 	$r;;==D8#####	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 		(                 s7   B%E-DED	ED	EE
Ez?header,header_first_partition_only,expected_first,expected_next)FFa,1
d,4
)TFr  r  )FTr  r  )TTr  r  aabbFzaa,bb
Tr  c                    d}t                               g dg dd          }t                              ||          }t	                      5 }|                    t          j                            |d          d| |           t          j                            |d	          }t          |          5 }	|	
                                }
|
|k    sJ 	 d d d            n# 1 swxY w Y   t          j        |           t          j                            |d
          }t          |          5 }	|	
                                }
|
|k    sJ 	 d d d            n# 1 swxY w Y   t          j        |           d d d            d S # 1 swxY w Y   d S )Nr   )r/  re   r0  r  r  r   )r4   r   r\   r  r   r]   r  rL  z	fooa*.csvF)r  rp   rI  z	fooa0.csvz	fooa1.csv)rF   rx   rw   r*  r    r  r5   r6   r,   r^  r  r  )rp   rI  expected_firstexpected_nextpartition_countrS   r   r&  r  r  r  s              r'   test_to_csv_headerr    s7    O	:::ASASASTT	U	UB
...
9
9C	 R 	

GLL[))(C	 	 	
 	
 	
 7<<K00(^^ 	*r;;==D>)))))	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 		(7<<K00(^^ 	)r;;==D=(((((	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 		('                 s\   A'F5CFC#	#F&C#	'AF-EFE	FE	FF
Fc                    t                               ddgi          } t                              | d          }ddh}t	                      5 }|                    t          j                            |d          dd	           t          j                            |d
          }t          |d          5 }|
                                }d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   ||v sJ d S )Nr  r   r4   rL  s   0
s   0
r  F)rp   r  r  rb)rF   rx   rw   r*  r    r  r5   r6   r,   r^  read)rS   r   r   r&  r  r   raws          r'   test_to_csv_line_endingr    sU   	sQCj	!	!B
...
+
+C&!H 
 R

27<<J//U
KKK7<<J//(D!! 	Q&&((C	 	 	 	 	 	 	 	 	 	 	 	 	 	 	              
 (??????s7   A'C+3CC+C	C+C	C++C/2C/block_listsr   r\   )r  r   r]   c                F    g | ]}t          t          |                    S r$   )r^   r   r%   r  s     r'   r(   r(     s$    +++AeAhh+++r)   c                    t          t          |                     }t          |          t          t          t          |                               k    sJ d S r   )r^   r   r`   r   )r  masks     r'   test_block_maskr    sN     
;''((Dt99D!5!56677777777r)   c            
        t                      5 } t          g d          D ]p\  }}t          t          j                            | t          |          dz             d          5 }|                    |           d d d            n# 1 swxY w Y   qt          	                    t          j                            | d          ddt          ig d                                          }t                              d	d
gddgddgddgd          }|d                             d          |d<   t          ||d           d d d            d S # 1 swxY w Y   d S )N)z0, 1, 2rt  z6, 7, 8r  r  *.csvTr6   )rL  rM  C)r   r   r   r   r]   r4   r   r   r7  z0.csvz2.csv)rL  rM  r  r6   r  Fr   )r    r  r^  r5   r6   r,   r.   r_  rw   rG   r7   r   rF   rx   rP   r   )rb  rA   r   filer   rS   s         r'   &test_reading_empty_csv_files_with_pathr    s   	 1T#$>$>$>?? 	$ 	$JAwbgll4Q&993?? $4

7###$ $ $ $ $ $ $ $ $ $ $ $ $ $ $GLLw'' $/!//	  
 

 ')) 	 \\VVV '*	 
 
 Z&&z226
&"%0000'1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1s7   AE$B:EB

EB
B2EEEc                   t           j                            t          |           d          }t                              dddg          }|                    |d           t                              |          }|	                    d          }t          ||                    d                                                     d S )Nr>  ro  r  )r}  barFrO  r}  )r5   r6   r,   r.   rF   rx   r  rw   rG   groupbyr   	get_groupr   )r    r6   rf  ddf1ddfss        r'   test_read_csv_groupby_get_groupr    s     7<<FZ00D
,,1--.
/
/CJJt5J!!!;;tD<<Dc4>>"%%--//00000r)   c                   t           j                            t          |           d          }t	          d          }t	          t          t          |                              }t                              d t          ||          D             g          }|
                    |           t	          d          }t                              |          |                             d          }t          ||         |           d S )Nr>  abcdefghijklmnopqrstuvwxyzc                    i | ]\  }}||	S r$   r$   )r%   r0  rB   s      r'   rC   z1test_csv_getitem_column_order.<locals>.<dictcomp>  s    ???$!QA???r)   hczzkylaaper4   )r5   r6   r,   r.   r^   r   r`   rF   rx   rr  r  rw   rG   r{   r   )r    r6   r_   r|   rf  df2s         r'   test_csv_getitem_column_orderr    s     7<<FZ00D/00G%G%%&&F
,,??#gv*>*>???@
A
ACJJt =!!G
++d

G
$
)
)!
,
,Cc'lC     r)   c                 
   t          t                    5 } t                              |           }||d         dk             dg         }t                              |           }||d         dk             dg         }t          |j        |j        g          }t          |d          }t          |t                    sJ t          |j                  ddhk    sJ t          ||           d d d            d S # 1 swxY w Y   d S )Nr#  g     i@r$  )r  zread-csv)r   r&  rF   rG   rw   r   ru   rl  r"   rv   r   r   r_   r   )r   expectr   dsksubgraph_rds        r'   &test_getitem_optimization_after_filterr    s1   	*		 
Rv./8kk"oo#f+%&w/(DDDZ00+'788888;&''FE?::::&#
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
s   CC88C<?C<c                $   t           j                            t          |           d          }d}t                              t          |                    }t          |d          5 }|                    |           d d d            n# 1 swxY w Y   t          j
        t          d          5  t                              |d           d d d            n# 1 swxY w Y   t                              |dd	          }t          ||           d S )
Nr>  s   a,b
1,"hi
"
2,"oi
"
r\  zEOF encounteredr   r   rh  r4   )ri  sample_rows)r5   r6   r,   r.   rF   rG   r   r^  r_  r   r   r   rw   r   )r    r6   r  r   r   rS   s         r'   test_csv_parse_failr  -  sX   7<<FZ00D'D{{74==))H	dD		 Q	              	z):	;	;	; % %
D$$$% % % % % % % % % % % % % % %	T"!	4	4Bb(s$   'B		BB/CCCc                B   dd l }ddlm} t          j                            t          |           d          }t          j                            t          |           d          }t          |d          5 }t          d          D ]_}|	                    |
                    dd          d	d
|
                    dd          d	d
|
                    dd          d	d           `	 d d d            n# 1 swxY w Y    |||           t          |d          5 }t          d          D ]_}|	                    |
                    dd          d	d
|
                    dd          d	d
|
                    dd          d	d           `	 d d d            n# 1 swxY w Y   t                              |d dt          d           }t                              |d dt          d           }|j                                        |j                                        k    sJ d S )Nr   )copyfilezold.csvnew_csvr  ro  r4   i ʚ;09z, r+   r/  r\   r8   )rp   r  rl   r   )randomshutilr  r5   r6   r,   r.   r^  r   r_  	randrangerw   rG   ru   r  )	r    r  r  old_csv_pathnew_csv_pathr   r  new_dfold_dfs	            r'   6test_csv_name_should_be_different_even_if_head_is_samer  :  s   MMM7<<FY77L7<<FY77L 
lC	 	  Ar 	 	AGG##Au--vvvF4D4DQ4N4NvvvU[UeUefginUoUovvvv   	               H\<((( 
lC	 	  Aq 	 	AGG##Au--vvvF4D4DQ4N4NvvvU[UeUefginUoUovvvv   	               [[TSt   F [[TSt   F ;!1!1!3!3333333s&   5A0C22C69C6A0FFFc                   d t          dd          D             d t          dd          D             d}t                              |          }t          |           dz   }t          d	          D ]}|                    | d
| dd            t
                              |dz   d          }t          |j        t          	                    |j        gd	z                       d S )Nc                    g | ]}|S r$   r$   r  s     r'   r(   z8test_select_with_include_path_column.<locals>.<listcomp>_  s    +++!+++r)   r   r  c                    g | ]}|S r$   r$   r  s     r'   r(   z8test_select_with_include_path_column.<locals>.<listcomp>_  s    5Q5Q5QAa5Q5Q5Qr)   r   )col1col2)r  /r]   file_r  FrO  r  Tr
  )
r   rF   rx   r.   r  rw   rG   r   r  rN   )r    r  rS   	temp_pathr  r   s         r'   $test_select_with_include_path_columnr  \  s     ,+U1c]]+++5Q5QsC5Q5Q5QRRA	1		BFc!I1XX ; ;
		Y,,Q,,,E	::::
++i')t+
D
DCch		27)a-0011111r)   	use_namesc                   t          d          }|r	g d}ddg}nd x}}t          j                            t	          |           d          }t
                              |d                               |dd           t
                              |d	||
          }t                              |d	||d          }t          ||d           d S )Nz    city1,1992-09-13,10
    city2,1992-09-13,14
    city3,1992-09-13,98
    city4,1992-09-13,13
    city5,1992-09-13,45
    city6,1992-09-13,64
    )cityr  salesr  r  z	input.csvr   Fr  r   )rp   r   rh   <   )rp   r   rh   r   r   )
r   r5   r6   r,   r.   rF   rG   r  rw   r   )r    r  r  r   rh   r6   rS   r   s           r'   test_names_with_header_0r  k  s     		 	C  )))7#7<<F[11DKKDK!!((U5(III	T!5'	B	BB
++   C b#5))))))r)   )
__future__r   rZ  r5   r  r  r   r   unittestr   r   r   rF   rw   fsspecfsspec.compressionr   packaging.versionr	   tlzr
   r   ru   	dask.baser   dask.bytes.corer   dask.bytes.utilsr   	dask.corer   dask.dataframe._compatr   r   r   dask.dataframe.io.csvr   r   r   r   r   dask.dataframe.optimizer   dask.dataframe.utilsr   r   r   r   dask.layersr   
dask.utilsr   r   r    r!   dask.utils_testr"   compression_fmtsr2   r7   r/   r   r@   tsv_text	tsv_text2r&  rH   r   	tsv_files	fwf_filesrT   rZ   r   r   csv_units_rowtsv_units_rowmarkparametrizerG   
read_tableread_fwfcsv_and_tablerf   ri   rm   rq   r}   r   r   r   r   r   r   
csv_blocks
tsv_blocksr   r   r   skipifr   r   r   r   r   r  r  r  r  r  r  r!  r)  r1  r6  r8  r@  rI  rW  slowrc  rf  rj  rn  rz  r   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  xfailxfail_pandas_100paramr  r  r  r  r  r	  r  r  r  r  r  r  r.  r<  r@  rG  rJ  rQ  rU  rW  r[  __version__ra  rd  rg  ri  rt  r{  r~  r  r  r  r  r  r   r  r  r  r  skip_with_pyarrow_stringsr  r  r  r  r  r$   r)   r'   <module>r     s9   " " " " " "  				                        V""V)**  $ $ $ $ $ $ % % % % % % % % % % % % % %  . . . . . . & & & & & & % % % % % %       C C C C C C C C C C              ? > > > > >            ) ( ( ( ( ( ; ; ; ; ; ; ; ; ; ; ; ; % % % % % % *)5)))TF2 < < <" " " 
EGG 	" C&& 
EGG 
" 
EGG  	O)O 	 HGY__5F5FGGG		"
 0	" 	" $      
 #%%dE22 ''	i 		"	i   $ $ $ . . . & & & $ $ $ B B B$ 
0 
0 
0   6 k2;	*R]BM9,UV 6 6	 6 k2;	*R]BM9,UV 
6 
6	 
6 k2;	*R]BM9,UV 6 6	 6 !	bk9m<	y-@ 	6 	6 	6 _-_-
  12 12
 r{J/"-1LM K K K r{J/"-1LM - - - 	bk8S1	x6	y&9 0 0 0 M   6 6 6 	bk8Q/	x!R9 C C C 	bk8Q/	x!R9 
F 
F 
F k2;	*R]BM9,UV 	4 	4	 	4 k2;	*R]BM9,UV  	  r{I.	0JK 
- 
- 
- r{I.	0JK 
- 
- 
- r{I.	0JK A A A r{I.	0JK 	1 	1 	1 r{I.	0JK 
1 
1 
1"$ $ $&$ $ $L L L" " "0 0 0(2 2 2$7 7 72Y Y Y<<+ <+ <+~ &(9::H H ;: H . . .< < <" " " tRj11 011
 
 21 21
. E E E*= = = 999--3 3 .-3 A A A* * *  "& & &P P P) ) )$  	" 	" 	"P P P" " "* * *s s sl +  +  +F
. 
. 
.  $	 	 	 ;$$,V$WW  X%5666[(8999   "7 7 75 5 5# # #8# # #"
$ 
$ 
$N N N %.OPP
X 
X QP
X
 
 
	T 	T 	T
 
 
:. . .*" " "<6  6  6 r" " "&	S 	S 	S  . . .E E E	" 	" 	" ?@@
. 
. A@
. GF77:#6#666   3 3	 3
3 
3 
3: : :6# # #1 1 1&: : :   
 
 
$   +k?-KLL  ML E(''&
ui3
tY0
 
 
 
:  & 
Q!iii 
	R!b1#++r+++	 8 8 8
1 1 1.
1 
1 
1! ! !  &  '&
 
 
4 4 4D2 2 2 tUm44$* $* 54$* $* $*r)   