o
    Nrf                     @  s  d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlmZmZmZmZ d dlmZ d dlZd dlZd dlmZmZmZ d dlZd dlmZmZmZ d dlmZ d dl m!Z! d d	l"m#Z# e#d
Z$e#dZ%e#dZ&e#dZ'e#dZ(e#dZ)e#dZ*ej+dddd Z,dd Z-dd Z.dd Z/ej01ddd Z2ej01ddd Z3ej01dd d! Z4ej01dd"d# Z5ej01dd$d% Z6ej01dd&d' Z7ej01dd(d) Z8ej01dd*d+ Z9ej01dd,d- Z:ej01dd.d/ Z;ej01dd0d1 Z<ej01dd2d3 Z=ej01dd4d5 Z>ej01dd6d7 Z?d8d9 Z@d:d; ZAd<d= ZBd>d? ZCd@dA ZDdBaEdCdD ZFG dEdF dFZGeHeIjJdGjJeGjKeG jKeGjLeG jLeGjMgZNej0OdHeF dIdJ ZPej0OdHeNdKdL ZQdMdN ZRej0jSdOdPej0OdQdRdgdSdT ZTdUdV ZUd dWdXZVddZd[ZWd\d] ZXd^d_ ZYd`da ZZej01dbdcdd Z[ej01dbdedf Z\ej01dbdgdh Z]ej01dbdidj Z^ej01dkdldm Z_ej01dbdndo Z`ej01dbdpdq Zaej01dbdrds Zbdtdu Zcdvdw Zddxdy Zedzd{ Zfd|d} Zgd~d Zhdd Zidd ZjG dd dZkG dd dekZldd Zmdd Zndd Zodd Zpdd Zqdd Zrdd Zsdd Ztdd Zudd Zvej0Odeeeegdd ZwejxG dd dZyejxG dd dZzejxG dd dZ{G dd dZ|dd Z}dd Z~ej0Odg ddd Zej01ddd Zej01ddd Zej0OddBddddddg di deIegdd Zdd Zej01dej0ddd Zej01dej0Odd¡ddĄ ZddƄ Zej0Odg dǢddɄ Zdd˄ Zdd̈́ Zddτ Zddф Zddӄ ZddՄ Zej01dddׄ Zej0Oddejdej01ddۍgdd݄ Zdd߄ Zej01ddd Zej01ddd Zej01ddd Zej01ddd Ze*srG dd dZe Z*e*jdddd Ze*jddddd Ze*jdddd Ze*jddddd Ze*dgddd ZeeeeegZej01dej0OdHedd Zej01ddd Zej01ddd ZdS (      )annotationsN)EnumFlagIntEnumIntFlag)Union)composecurrypartial)TokenizationErrornormalize_tokentokenize)literal)tmpfile)import_or_nonez
dask.arrayzdask.dataframenumpyzscipy.sparseZpyarrowZpandasnumbaT)Zautousec                  c  s    ddl m} m} tt |   W d   n1 sw   Y  tt |  W d   n1 s6w   Y  dV  tt |   W d   n1 sRw   Y  tt |  W d   dS 1 slw   Y  dS )zdTest that tokenize() and normalize_token() properly clean up context
    variables at all times
    r   _ensure_deterministic_seenN)	dask.baser   r   pytestraisesLookupErrorgetr    r   a/var/www/html/software/conda/envs/catlas/lib/python3.10/site-packages/dask/tests/test_tokenize.pycheck_contextvars#   s   



"r   c            
      O  s   t jddiU t| i |}t| i |}||ksJ tt| |f\}}tt| |f\}}tt||f\}}t|i |}t|i |}	||	ksUJ W d    |S 1 s`w   Y  |S )Ntokenize.ensure-deterministicT)daskconfigsetr   cloudpickleloadsdumps)
argskwargsbeforeafterZargs2Zkwargs2Zargs3Zkwargs3Ztok2Ztok3r   r   r   check_tokenize5   s   	
r)   c                    s   t d tdt dksJ G  fddd   } tt t |  W d    n1 s.w   Y  G dd d}| }tt t | W d    d S 1 sRw   Y  d S )N{   c                      s(   e Zd Zdd Z fddZdd ZdS )ztest_check_tokenize.<locals>.Ac                 S  s   t   | _d S N)randomtokselfr   r   r   __init__Z   s   z'test_check_tokenize.<locals>.A.__init__c                       dfS Nr   r   r.   Ar   r   
__reduce__]      z)test_check_tokenize.<locals>.A.__reduce__c                 S     | j S r+   )r-   r.   r   r   r   __dask_tokenize__`      z0test_check_tokenize.<locals>.A.__dask_tokenize__N)__name__
__module____qualname__r0   r5   r8   r   r3   r   r   r4   Y   s    r4   c                   @     e Zd Zdd ZdS )ztest_check_tokenize.<locals>.Bc                 S  s   t   S r+   )r,   r.   r   r   r   r8   i   r6   z0test_check_tokenize.<locals>.B.__dask_tokenize__Nr:   r;   r<   r8   r   r   r   r   Bh       r?   )r)   r   r   r   AssertionError)ar?   br   r3   r   test_check_tokenizeR   s   


"rD   c                  C  s   d} t t| tsJ d S )N         )
isinstancer   strrB   r   r   r   test_tokenizeq   s   rL   znot npc                   C  s   t tdt dksJ t tdt tdksJ t tdt tdks-J t tdt dks:J t tdt tdksJJ d S )NrF   1)r)   npint64int32Zuint32float64r   r   r   r   test_tokenize_scalarv   s
     $rR   c                   C  s4   t tjddt tjddksJ d S )N    )r)   rN   r,   RandomStateZrandom_sampler   r   r   r   .test_tokenize_numpy_array_consistent_on_values   s
   rV   c                   C  s   t tjdjdd d S )N   i2dtype)r)   rN   r,   astyper   r   r   r   /test_tokenize_numpy_array_supports_uneven_sizes   s   r\   c                   C  s   t tjdd d d  d S )N   rG   )r)   rN   r,   r   r   r   r   'test_tokenize_discontiguous_numpy_array   s   r^   c                   C  s   t tjdgdd d S )Nz2000-01-01T12:00:00zM8[ns]rY   r)   rN   arrayr   r   r   r   test_tokenize_numpy_datetime   s   ra   c                   C  sh   t tjdddt tjdddksJ t tjdgddgdd t tjdgddgdd ks2J d S )N      ?Zf8rY   rF   rG   )rB   i4)rC   i8r   r_   r   r   r   r   test_tokenize_numpy_scalar   s   
rf   c                   C  sZ   t jddd id tt dtt dksJ W d    d S 1 s&w   Y  d S )Nallc                 S     dS Nfoor   xr   r   r   <lambda>       z7test_tokenize_numpy_scalar_string_rep.<locals>.<lambda>)	formatterrF   rG   )rN   Zprintoptionsr)   r`   r   r   r   r   %test_tokenize_numpy_scalar_string_rep   s   ""rp   c                  C  s   t jg dtd} t| t| ksJ tt jg dtdtt jg dtdks+J tt jg dtdtt jg dtdksCJ G dd d}t jdd | gtd}t| d S )N)rB   ZaaaaarY   )rB   Nrq   ))rF   rB   )rF   N)rF   rq   c                   @     e Zd ZdS )zCtest_tokenize_numpy_array_on_object_dtype.<locals>.NeedsCloudPickleNr:   r;   r<   r   r   r   r   NeedsCloudPickle       rt   rB   )rN   r`   objectr)   )rB   rt   rl   r   r   r   )test_tokenize_numpy_array_on_object_dtype   s   
rw   c                  C  sR   t g } | js
J t| t| ksJ t jg t  d}t| t|ks'J d S )NrY   )rN   r`   stridesr)   rO   )arrZarr2r   r   r   test_empty_numpy_array   s
   

rz   c                 C  s  t | d}t|d}|d W d    n1 sw   Y  t|d}tj|tjdddd}tj|tjdddd}tj|tjdddd}t|t|ksRJ t|t|ks\J t|t|ksfJ t|d	d
 t|d	d
 ksxJ t|d	d
 t|d	d
 ksJ t|d	d t|d	d ksJ t|d	d t|d	d ksJ t|d	d t|d	d ksJ |d	d
 }|d	d
 }t|t|ksJ W d    d S 1 sw   Y  d S )NZ	demo_datawbs
   ashekwichtrbrr      )rZ   modeoffsetshaperF   rG   rH   )rJ   joinopenwriterN   ZmemmapZuint8r)   )ZtmpdirfnfZmmap1Zmmap2Zmmap3Zsub1Zsub2r   r   r   !test_tokenize_numpy_memmap_offset   s(   $$$$$"r   c                  C  s  t d} td}t| | ttj| dd}W d    n1 s#w   Y  t d} td}t| | ttj| dd}W d    n1 sKw   Y  t|t|ksZJ ||ks`J t dz} tjjdd}t| | tj| dd}tj| dd}t|dd d f }t|dd d f }	t|dd	d d f }
t|d d df }t||	|
|hd
ksJ t|t|ksJ t|dd d f t|dd d f ksJ W d    d S 1 sw   Y  d S )N.npyr~   r}   Z	mmap_mode)
   r   )sizer   rF   rH      )	r   rN   arangesaver)   loadr,   normallen)r   x1yZx2zrl   mmZmm2rB   rC   cdr   r   r   test_tokenize_numpy_memmap   s2   




."r   c               	   C  s   t dK} t d.}td}t| | t|| tj| dd}|| }t|t|ks0J W d    n1 s:w   Y  W d    d S W d    d S 1 sRw   Y  d S )Nr   r~   r}   r   )r   rN   r   r   r   r)   )Zfn1Zfn2rl   rB   rC   r   r   r   &test_tokenize_numpy_memmap_no_filename   s   
Pr   c                  C  s   t tjt tjksJ tj} tj}t| tjsJ t|tjs"J t | t |ks,J tjdd dd}tjdd dd}tjdd dd}t |t |ksTJ t |t |ks^J d S )Nc                 S     | d S NrF   r   rk   r   r   r   rm         z+test_tokenize_numpy_ufunc.<locals>.<lambda>rF   c                 S  r   r   r   rk   r   r   r   rm     r   c                 S  r   NrG   r   rk   r   r   r   rm     r   )r)   rN   sincosrI   Zufuncda
frompyfunc)Znp_ufuncZ	np_ufunc2incZinc2Zinc3r   r   r   test_tokenize_numpy_ufunc  s   r   c               	   C  s,  t dd dd} tjddi t| t| ksJ t| t| ks&J W d    n1 s0w   Y  tjddi# tjt	dd t|  W d    n1 sTw   Y  W d    n1 scw   Y  t| dd	t| dd	ksvJ tjt	d
d t| dd	 W d    d S 1 sw   Y  d S )Nc                 S  r   r   r   rk   r   r   r   rm     r   z;test_normalize_numpy_ufunc_unserializable.<locals>.<lambda>rF   r   FTz,Cannot tokenize.*dask\.array\.ufunc.*insteadmatchZensure_deterministiczCannot tokenize)
rN   r   r   r    r!   r   r   r   r   r   )r   r   r   r   )test_normalize_numpy_ufunc_unserializable  s&   
"r   c               	   C  s.  G dd d} |  }t jddi t|t|ksJ t|t|ks'J W d    n1 s1w   Y  t jddi# tjtdd t| W d    n1 sUw   Y  W d    n1 sdw   Y  t|ddt|ddkswJ tjtdd t|dd W d    d S 1 sw   Y  d S )	Nc                   @  r=   )z/test_normalize_object_unserializable.<locals>.Cc                 S     J r+   r   r.   r   r   r   r5   1     z:test_normalize_object_unserializable.<locals>.C.__reduce__Nr:   r;   r<   r5   r   r   r   r   C0  r@   r   r   FTz"cannot be deterministically hashedr   r   )r   r    r!   r   r   r   r   r   )r   r   r   r   r   $test_normalize_object_unserializable/  s(   
"r   c                  C  sV   t tttd} t tttd}t ttdd}t| t|ksJ t| t|ks)J d S )N)r   r~   )r
   f3f2f1r)   )r   ghr   r   r   1test_tokenize_partial_func_args_kwargs_consistentJ  s
   r   c               
   C  sJ   dddt dddtdtddd	td
fD ]
} t| | u s"J qd S )NrF   g?rM   rG   rH   z1.1        z
/this/that)slicedecimalDecimaldatetimedatepathlibPurePathr   )ir   r   r   test_normalize_baseR  s   
	r   c               	   C  s  t jddi: t } t| t| ksJ tt tt ks"J tdd tdD dks1J t| t| ks;J W d    n1 sEw   Y  t jddi> tj	t
dd t|  W d    n1 siw   Y  tj	t
dd t|  W d    n1 sw   Y  W d    n1 sw   Y  t| dd	t| dd	ksJ tj	t
dd t| dd	 W d    d S 1 sw   Y  d S )
Nr   Fc                 S  s   h | ]}t t qS r   r   rv   ).0_r   r   r   	<setcomp>j  s    z'test_tokenize_object.<locals>.<setcomp>d   TZdeterministicr   r   )r   r    r!   rv   r   r   ranger   r   r   r   )or   r   r   test_tokenize_object_  s,   

"r   c                  C  sh   G dd d} t |  ddt |  ddksJ tt t |   W d   dS 1 s-w   Y  dS )zdTest that the ensure_deterministic override is not lost if tokenize() is
    called recursively
    c                   @  r=   )z/nested_tokenize_ensure_deterministic.<locals>.Cc                 S  s
   t t S r+   r   r.   r   r   r   r8        
zAnested_tokenize_ensure_deterministic.<locals>.C.__dask_tokenize__Nr>   r   r   r   r   r     r@   r   Fr   N)r   r   r   r   r   r   r   r   $nested_tokenize_ensure_deterministic~  s   "r   rF   c                    s   dd dd dd dd g} | d d \ dd }d	d
 }| ||g7 } d fdd}|  | dd }|  |  fdd}|  | fdd}|  | dd }|  | dd }|  |  fdd}|  | fdd}|  | | S )Nc                 S     | S r+   r   rk   r   r   r   rm     rn   z"_local_functions.<locals>.<lambda>c                 S  r   r   r   rk   r   r   r   rm     r   c                 S  r   r+   r   r   r   r   r   rm     rn   c                 S  r   r   r   r   r   r   r   rm     r   rG   c                 S  r   r+   r   rk   r   r   r   func  r   z_local_functions.<locals>.funcc                 S  r   r+   r   rk   r   r   r   r     r   z_local_functions.<locals>.f2rF   c                     s   d7  S r   r   r   )rB   local_scoper   r   r     s   c                   S  s   t d7 a t S r   )_GLOBALr   r   r   r   r     s   c                 S     || S r+   r   rl   r   r   r   r   r     r6   c                 S  r   r+   r   r   r   r   r   r     r6   c                 S     dd }|| S )Nc                 S  r   r   r   rk   r   r   r   rm     r   0_local_functions.<locals>.func.<locals>.<lambda>r   r   r   r   r   r        c                 S  r   )Nc                 S  r   )NrH   r   rk   r   r   r   rm     r   r   r   r   r   r   r   r     r   c                       }|| S r+   r   r   rK   r   r   r        c                   r   r+   r   r   rC   r   r   r     r   )append)	all_funcsr   r   r   )rB   rC   r   r   _local_functions  s6   







r   c                   @  s    e Zd Zdd Zedd ZdS )WithClassMethodc                 C     d S r+   r   r.   r   r   r   r     r   zWithClassMethod.fc                 C  r   r+   r   clsr   r   r   r        zWithClassMethod.gN)r:   r;   r<   r   classmethodr   r   r   r   r   r     s    r   rj   r   c                 C     t |  d S r+   r)   r   r   r   r   test_tokenize_local_functions     r   c                 C  r   r+   r   r   r   r   r   test_tokenize_special_callables  r   r   c                  C  s4   t  t } dd | D }tt|t|ksJ d S )Nc                 S     g | ]}t |qS r   r   r   r   r   r   r   
<listcomp>      z8test_tokenize_functions_unique_token.<locals>.<listcomp>)r   _special_callablesr   r!   )r   tokensr   r   r   $test_tokenize_functions_unique_token  s   
r   z3https://github.com/cloudpipe/cloudpickle/issues/453)reasoninstanceFc                   s(    fdd}t | t | ksJ d S )Nc                    s   G dd d}  r|  S | S )Nc                   @  rr   )zItest_tokenize_local_classes_from_different_contexts.<locals>.f.<locals>.CNrs   r   r   r   r   r     ru   r   r   r   r   r   r   r     s   z>test_tokenize_local_classes_from_different_contexts.<locals>.fr   )r   r   r   r   r   3test_tokenize_local_classes_from_different_contexts  s   r   c                  C  s$   dd } t |  t |  ksJ d S )Nc                  S  s   dd } | S )Nc                   S  rh   )Nr*   r   r   r   r   r   r     r   zKtest_tokenize_local_functions_from_different_contexts.<locals>.f.<locals>.gr   )r   r   r   r   r        z@test_tokenize_local_functions_from_different_contexts.<locals>.fr   )r   r   r   r   5test_tokenize_local_functions_from_different_contexts  s   r   c                 C  r   r+   r   rB   rC   r   r   r   r   r     r   r   rG   c                 C  r   r+   r   r   r   r   r   r     r   r   c                 C  r   r+   r   rK   r   r   r   r     r   r   c                   C  s   t tt tks
J d S r+   )r)   r   r   r   r   r   r   test_tokenize_callable#  s   r   c                   C  s   t ttddt ttddksJ t ttddt ttddks$J t tttt tttks4J t ttt ttksBJ t ttddt ttddksTJ d S )NrG   r   rH   rF   )r)   r
   r   r   r   r   r	   r   r   r   r   !test_tokenize_composite_functions'  s
   $$ (r   znot pdc                  C  s   t jg dg ddg dd} t jg dg ddg dd}t| t|ks*J d|j_t| t|ks8J t g dg dd} t g dg dd}| jd| d< t| t|ksbJ | jd|d< t| t|kstJ d S )	NrE   )4ZasdNrl   r   indexrj   )rB   rC   rB   categoryr   )pd	DataFramer)   r   namer   r[   rB   rC   r   r   r   test_tokenize_pandas/  s     r   c                  C  s,   t jg dg ddg dd} t|  d S )NrE   )r   u   asdN)u   xu   yr   r   r   r)   dfr   r   r   $test_tokenize_pandas_invalid_unicode@  s   r  c                  C  s8   t jd g dddd d gig dd} t|  d S )N   örE   r   )r   r   encoder)   r  r   r   r   (test_tokenize_pandas_mixed_unicode_bytesI  s
   r  c                  C  s0   G dd d} t ddd |  gi}t| d S )Nc                   @  rr   )z:test_tokenize_pandas_cloudpickle.<locals>.NeedsCloudPickleNrs   r   r   r   r   rt   T  s    rt   rl   rj   r  )rt   r  r   r   r    test_tokenize_pandas_cloudpickleR  s   r  znot ddc                  C  s   t jg dddt jdgddt jg dddt jt dgddt jt jdd	d
gddt jddgt jjjg ddddg} | t jg dddt jg dddg | D ]}t| qWd S )N)rF   r   NZInt64rY   2000z	Period[D])rF   r   r   zSparse[int]zdatetime64[ns]ZCET)tzzdatetime64[ns, CET]rB   rC   r   F)Zordered)rB   rC   Nstring)TFNboolean)r   r`   	TimestampapitypesZCategoricalDtypeextendr)   )Zarraysry   r   r   r   $test_tokenize_pandas_extension_array\  s$   
r  c                   C  s   t tj d S r+   )r)   r   ZNAr   r   r   r   test_tokenize_nau  s   r  c               	   C  s\   t jdt jdt jdt jdt jddt jddt jddfD ]} t|  q%d S )NrF   rG   )ZyearsrW   )monthsr   days)r   offsetsZSecondZ
MonthBeginZDayZBQuarterEndZ
DateOffsetr)   )r   r   r   r   test_tokenize_offsetz  s   







	r  c                  C  s:   t ddg} t|  t jddgddgg} t|  d S )NrB   rC   r   rF   )r   Indexr)   Z
MultiIndexZfrom_product)idxr   r   r   test_tokenize_pandas_index  s   r  c                   C  s~   t ddd t dt dddksJ t dddt dddks J t dddt dddks.J t dddt dddiks=J d S )	Nr~   rF   rk   rG   r   bar)rj   rj   r   r   r   r   r   test_tokenize_kwargs  s
   "r  c                  C  .   G dd d} t | dt | dksJ d S )Nc                   @     e Zd Zdd Zdd ZdS )z$test_tokenize_same_repr.<locals>.Fooc                 S  
   || _ d S r+   rk   r/   rl   r   r   r   r0     r   z-test_tokenize_same_repr.<locals>.Foo.__init__c                 S  rh   )Nza foor   r.   r   r   r   __repr__  r   z-test_tokenize_same_repr.<locals>.Foo.__repr__N)r:   r;   r<   r0   r!  r   r   r   r   Foo      r"  rF   rG   r   r"  r   r   r   test_tokenize_same_repr  s    r%  c                  C  r  )Nc                   @  s   e Zd ZdZdd ZdS )z"test_tokenize_slotted.<locals>.Foork   c                 S  r  r+   rk   r   r   r   r   r0     r   z+test_tokenize_slotted.<locals>.Foo.__init__Nr:   r;   r<   	__slots__r0   r   r   r   r   r"    s    r"  rF   rG   r   r$  r   r   r   test_tokenize_slotted  s    r(  c                  C  s<   G dd d} t | ddt | ddksJ t |   d S )Nc                   @  s   e Zd ZdZdddZdS )z+test_tokenize_slotted_no_value.<locals>.Foor   Nc                 S  s$   |d ur|| _ |d ur|| _d S d S r+   r   r/   rl   r   r   r   r   r0     s
   
z4test_tokenize_slotted_no_value.<locals>.Foo.__init__)NNr&  r   r   r   r   r"    s    r"  rF   rk   r   r   r$  r   r   r   test_tokenize_slotted_no_value  s    	r*  c                  C  s   G dd d} G dd d| }|ddj ddiksJ t|ddt|ddt|dd t|ddg}tt|t|ksAJ d S )	Nc                   @     e Zd ZdZdS )z)test_tokenize_slots_and_dict.<locals>.Foork   N)r:   r;   r<   r'  r   r   r   r   r"        r"  c                   @  r=   )z)test_tokenize_slots_and_dict.<locals>.Barc                 S  s   || _ |d ur|| _d S d S r+   r   r)  r   r   r   r0     s   
z2test_tokenize_slots_and_dict.<locals>.Bar.__init__Nr:   r;   r<   r0   r   r   r   r   Bar  r@   r.  rF   rG   r   rH   )__dict__r)   r   r!   )r"  r.  r   r   r   r   test_tokenize_slots_and_dict  s   r0  c                  C  s   G dd d} | d| d}}t |t |ksJ t |jt |jks&J t |}t| dd  t |}||ks<J tj| = d S )Nc                   @  $   e Zd Zdd Zdd Zdd ZdS )z!test_tokenize_method.<locals>.Fooc                 S  r  r+   rk   r   r   r   r   r0     r   z*test_tokenize_method.<locals>.Foo.__init__c                 S  r7   r+   rk   r.   r   r   r   r8     r9   z3test_tokenize_method.<locals>.Foo.__dask_tokenize__c                 S  rh   )NzHello worldr   r.   r   r   r   hello  r   z'test_tokenize_method.<locals>.Foo.helloN)r:   r;   r<   r0   r8   r2  r   r   r   r   r"    s    r"  rF   rG   c                 S  s
   | j d S r   rk   r.   r   r   r   rm     s   
 z&test_tokenize_method.<locals>.<lambda>)r)   r2  r   registerZ_lookup)r"  rB   rC   r'   r(   r   r   r   test_tokenize_method  s   
r4  c                  C  sf   G dd d} G dd d| }| d| d|d}}}t |t |ks'J t |t |ks1J d S )Nc                   @  r  )z'test_tokenize_callable_class.<locals>.Cc                 S  r  r+   rk   r   r   r   r   r0     r   z0test_tokenize_callable_class.<locals>.C.__init__c                 S  r7   r+   rk   r.   r   r   r   __call__  r9   z0test_tokenize_callable_class.<locals>.C.__call__N)r:   r;   r<   r0   r5  r   r   r   r   r     r#  r   c                   @  rr   )z'test_tokenize_callable_class.<locals>.DNrs   r   r   r   r   D  ru   r6  rF   rG   r   )r   r6  rB   rC   r   r   r   r   test_tokenize_callable_class  s
   r7  c                  C  sR   G dd d} t | ddt | ddksJ t | ddt | ddks'J dS )z(Always use ___dask_tokenize__ if presentc                   @  r1  )z<test_tokenize_callable_class_with_tokenize_method.<locals>.Cc                 S  s   || _ || _d S r+   r   r)  r   r   r   r0        
zEtest_tokenize_callable_class_with_tokenize_method.<locals>.C.__init__c                 S  r7   r+   rk   r.   r   r   r   r8     r9   zNtest_tokenize_callable_class_with_tokenize_method.<locals>.C.__dask_tokenize__c                 S  r   r+   r   r.   r   r   r   r5    r   zEtest_tokenize_callable_class_with_tokenize_method.<locals>.C.__call__N)r:   r;   r<   r0   r8   r5  r   r   r   r   r     s    r   rF   rG   rH   Nr   r   r   r   r   1test_tokenize_callable_class_with_tokenize_method  s    $r9  c                   @  s<   e Zd Zdd Zdd Zdd Zedd Zed	d
 Z	dS )HasStaticMethodsc                 C  r  r+   rk   r   r   r   r   r0     r   zHasStaticMethods.__init__c                 C  s   t t| | jfS r+   )r   typerl   r.   r   r   r   r8     s   z"HasStaticMethods.__dask_tokenize__c                 C  r   r+   r   r.   r   r   r   normal_method  r   zHasStaticMethods.normal_methodc                   C  r   r+   r   r   r   r   r   static_method  r   zHasStaticMethods.static_methodc                 C  r   r+   r   r   r   r   r   class_method  r   zHasStaticMethods.class_methodN)
r:   r;   r<   r0   r8   r<  staticmethodr=  r   r>  r   r   r   r   r:    s    
r:  c                   @  rr   )HasStaticMethods2Nrs   r   r   r   r   r@    ru   r@  c                  C  s   t dt dtd} }}t| jt|jksJ t| jt|jks&J t| jt|jks2J t| jt|jks>J t| jt|jksJJ t| jt|jksVJ d S )NrF   rG   )r:  r@  r)   r<  r=  r>  r   r   r   r   test_staticmethods"  s   rA  c               	   C  s   t dgt dgksJ t dgt dksJ t dgt dgks#J d} | | | dgg}t|ddddgfdddddd	gfgfgfksEJ d S )
NrF   rG   rF   rc   )rG   rH   listtuple)__seenr   rH   )r)   r   r   r   r   r   test_tokenize_sequences.  s   
rF  c                  C  sV   g d} G dd d}|| || }}t | ||  |jsJ t |t |ks)J dS )zuTest that calling tokenize() recursively doesn't alter the output due to
    memoization of already-seen objects
    rE   c                   @  r  )z$test_nested_tokenize_seen.<locals>.Cc                 S  s   || _ d | _d S r+   )rl   r-   r   r   r   r   r0   H  r8  z-test_nested_tokenize_seen.<locals>.C.__init__c                 S  s   | j s	t| j| _ | j S r+   )r-   r   rl   r.   r   r   r   r8   L  s   z6test_nested_tokenize_seen.<locals>.C.__dask_tokenize__N)r:   r;   r<   r0   r8   r   r   r   r   r   G  s    r   N)r)   r-   )r   r   c1c2r   r   r   test_nested_tokenize_seenA  s   

rI  c                   C  sd   t dddt dddksJ t dddt dddks J t dddt dddks0J d S )NrF   rl   )rl   rF   rF   rl   )rl   rG   rG   r   r   r   r   r   test_tokenize_dictW  s     $rK  c                   C  s    t h dt h dksJ d S )N>   rF   rG   rl   rJ  r   r   r   r   r   test_tokenize_set^  s   rL  c                  C  s\   ddl m}  | ddg}| ddg}| ddg}t|t|ks"J t|t|ks,J d S )Nr   )OrderedDict)rB   rF   )rC   rG   )collectionsrM  r)   )rM  rB   rC   r   r   r   r   test_tokenize_ordered_dictd  s   rO  c                  C  s$   G dd d} t |  |  d d S )Nc                   @  r  )z7test_tokenize_dict_doesnt_call_str_on_values.<locals>.Cc                 S  rh   )Nr   r   r.   r   r   r   r8   q  r   zItest_tokenize_dict_doesnt_call_str_on_values.<locals>.C.__dask_tokenize__c                 S  r   r+   r   r.   r   r   r   r!  t  r   z@test_tokenize_dict_doesnt_call_str_on_values.<locals>.C.__repr__N)r:   r;   r<   r8   r!  r   r   r   r   r   p  r#  r   )rF   2r   r   r   r   r   ,test_tokenize_dict_doesnt_call_str_on_valueso  s   rQ  c                  C  sD   d} | | d}| | d}dt t|v sJ t|t|ks J dS )zWhen sequence values are repeated, the 2nd+ entry is tokenized as (__seen, 0).
    This makes it important to ensure that dicts are sorted *before* you call
    normalize_token() on their elements.
    rE   rc   )rG   rF   rE  N)rJ   r   r)   )vZd1Zd2r   r   r   (test_tokenize_sorts_dict_before_seen_mapz  s
   

rS  c                    s`   d  fddt dD }  fddtt dD }dtt| v s$J t| t|ks.J dS )zSame as test_tokenize_sorts_dict_before_seen_map, but for sets.

    Note that this test is only meaningful if set insertion order impacts iteration
    order, which is an implementation detail of the Python interpreter.
    rE   c                      h | ]}| fqS r   r   r   r   rR  r   r   r     r   z:test_tokenize_sorts_set_before_seen_map.<locals>.<setcomp>r   c                   rT  r   r   rU  rV  r   r   r     r   rE  N)r   reversedrJ   r   r)   )s1s2r   rV  r   'test_tokenize_sorts_set_before_seen_map  s
   rZ  c                   C  sL   t tjddt tjddksJ t tjddt tjddks$J d S )NrF   r  rG   )r)   r   	timedeltar   r   r   r   test_tokenize_timedelta  s   


r\  	enum_typec                 C  sD   G dd d| }t |jt |jksJ t |jt |jks J d S )Nc                   @  s   e Zd ZdZdZdS )z!test_tokenize_enum.<locals>.ColorrF   rG   N)r:   r;   r<   REDBLUEr   r   r   r   Color  s    r`  )r)   r^  r_  )r]  r`  r   r   r   test_tokenize_enum  s   ra  c                   @     e Zd ZU ded< dS )
ADataClassintrB   Nr:   r;   r<   __annotations__r   r   r   r   rc       
 rc  c                   @  rb  )
BDataClassfloatrB   Nre  r   r   r   r   rh    rg  rh  c                   @  s"   e Zd ZU ejddZded< dS )NoValueDataClassF)initrd  rB   N)r:   r;   r<   dataclassesfieldrB   rf  r   r   r   r   rj    s   
 rj  c                   @  s   e Zd ZdddZdS )GlobalClassreturnNonec                 C  r  r+   )val)r/   rq  r   r   r   r0     r   zGlobalClass.__init__N)ro  rp  r-  r   r   r   r   rn    s    rn  c                    s   G dd d} G  fddd G dd d}G dd dt }tt d	 tt d	tt d
ks3J t|   t|d	 t| t  ksJJ d S )Nc                   @  r+  )z%test_local_objects.<locals>.LocalTyper  N)r:   r;   r<   rj   r   r   r   r   	LocalType  r,  rr  c                      s   e Zd Z fddZdS )z*test_local_objects.<locals>.LocalReduciblec                   r1   r2   r   r.   LocalReducibler   r   r5     r6   z5test_local_objects.<locals>.LocalReducible.__reduce__Nr   r   rs  r   r   rt    s    rt  c                   @  r=   )z-test_local_objects.<locals>.LocalDaskTokenizec                 S  rh   ri   r   r.   r   r   r   r8     r   z?test_local_objects.<locals>.LocalDaskTokenize.__dask_tokenize__Nr>   r   r   r   r   LocalDaskTokenize  r@   ru  c                   @  rr   )z&test_local_objects.<locals>.LocalChildNrs   r   r   r   r   
LocalChild  ru   rv  rF   rG   )rn  r)   )rr  ru  rv  r   rs  r   test_local_objects  s   
rw  c                  C  s   t d} t d}t|  t| t|ksJ td}tt ttks$J t| t|ks.J G dd dt }t|s=J tt t|ksGJ t|dt| ksSJ tddtttf fg}t| t|dkslJ t	 }t| d S )NrF   rG   c                   @  rr   )z%test_tokenize_dataclass.<locals>.SubANrs   r   r   r   r   SubA  ru   rx  rc  rB   )
rc  r)   rh  rl  is_dataclassmake_dataclassr   rd  rJ   rj  )a1a2b1rx  ZADataClassRedefinedDifferentlynvr   r   r   test_tokenize_dataclass  s"   r  other))rF   r   rG   )r~      rG   )r~   r   rF   c                 C  s$   t tdddt t|  ksJ d S )Nr~   r   rG   )r)   r   )r  r   r   r   test_tokenize_range  s   $	r  c                  C  s8   t d} t d}d|d< t| gt|gksJ d S )Ni  r   rT   )rN   r   r)   r   r   r   r   test_tokenize_numpy_array  s   

r  c                  C  s"   t jddt jgdd} t|  d S )Nrj   u   JoséOrY   )rN   r`   nanr)   rK   r   r   r   $test_tokenize_object_array_with_nans  s   r  rl   rB      arb   y              ?r   c                 C  r   r+   r   rk   r   r   r   test_tokenize_base_types  s   r  c                   C  s(   t tddgt tddgksJ d S )Nrl   rF   rG   )r)   r   r   r   r   r   test_tokenize_literal  s   (r  z+ignore:the matrix:PendingDeprecationWarningc                  C  s\   t jd} t | d}| }t|t|ksJ d|d d< t|t|ks,J d S )NrS   r   rF   r   )rN   r,   rU   Zasmatrixrandcopyr)   )rngrB   rC   r   r   r   test_tokenize_numpy_matrix  s   r  znot spcls_name)dokc                 C  s   t jd}tjdd|d| }| }t|t|ksJ t|dr,d|j	d d< n| dkr5d|d< nt
t| t|t|ksEJ | d	}t d|jd d< || }t|t|kseJ d S )
NrS   r   r   )Zrandom_statedatarF   r  )rH   rH   Zcoo)rN   r,   rU   spr  Zasformatr  r)   hasattrr  
ValueErrorr   row)r  r  rB   rC   r   r   r    test_tokenize_dense_sparse_array  s   


r  c                  C  s   ddg} | | d< ddg}||d< t | t |ksJ g g g}|d |d  |d |d  g g g}|d |d  |d |d  t |t |ksPJ i }||d< t | d S )NrF   rG   r   rH   )r)   r   )rB   rC   r   r   er   r   r    test_tokenize_circular_recursion:  s   r  ))i  r   r   )r   rW   r   )r   r      c                 C  s0   t ddd}t j|  }t|t|ksJ d S )Nr   r   r   )r   r   r)   )r  rB   rC   r   r   r   test_tokenize_datetime_dateT  s   	
r  c                	   C  sf  t tddddtjj t tdddd t tddd t tdd t tddddtjjt tddddtjjksDJ t tddddtjjt tddddtjjks`J t tddddtjjt tddddtjjks|J t tddddtjjt tddddtjjksJ t tddddtjjt tddddksJ d S )NrF   rG   rH   r   r~   )r)   r   timetimezoneutcr   r   r   r   test_tokenize_datetime_timeb  s:   r  c                  C  sr  g d} ddddt jjg}tt|d D ]}| |d |  }tt j |  qtt  dddddddt jjtt  dddddddt jjksIJ tt  dddddddt jjtt  dddddddt jjkskJ tt  dddddddt jjtt  dddddddt jjksJ tt  dddddddt jjtt  dddddddt jjksJ tt  dddddddt jjtt  dddddddt jjksJ tt  dddddddt jjtt  dddddddt jjksJ tt  dddddddt jjtt  dddddddt jjksJ tt  dddddddt jjtt  dddddddd ks7J d S )	NrE   r   r~   r   rW   rF   rG   rH   )r   r  r  r   r   r)   )requiredoptionalr   r%   r   r   r   test_tokenize_datetime_datetime~  sZ   
r  c                  C  s(   d} t tjdt| g}|  d S )Na  
    def inc(x):
        return x + 1

    inc2 = inc
    def sum(x, y):
        return x + y

    from dask.base import tokenize
    assert tokenize(inc) != tokenize(sum)
    # That this is an alias shouldn't matter
    assert tokenize(inc) == tokenize(inc2)

    def inc(x):
        return x + 1

    assert tokenize(inc2) != tokenize(inc)

    def inc(y):
        return y + 1

    assert tokenize(inc2) != tokenize(inc)

    def inc(x):
        y = x
        return y + 1

    assert tokenize(inc2) != tokenize(inc)
    z-c)
subprocessrunsys
executabletextwrapdedentcheck_returncode)scriptprocr   r   r   test_tokenize_functions_main  s   r  c                  C  sR   t jddtt jddfgddd id} | d	| d
}}t|t|ks'J d S )Nr4   paramF)reprr8   c                 S  r7   r+   )r  r.   r   r   r   rm     s    z7test_tokenize_dataclass_field_no_repr.<locals>.<lambda>)	namespacerF   rG   )rl  rz  ri  rm  r)   )r4   r{  r|  r   r   r   %test_tokenize_dataclass_field_no_repr  s   
r  c                   C  s   t tjt tjksJ dS )ziTop-level functions in the operator module have a __self__ attribute, which is
    the module itself
    N)r)   operatoraddmulr   r   r   r   test_tokenize_operator  s   r  c                  C  sf   t d} t d}t d}t| t|ksJ t| t|ks#J |    t| t|ks1J d S Nr*   i  r,   Randomr)   r   r   r   r   test_tokenize_random_state  s   


r  c                  C  sl   t jd} t jd}t jd}t| t|ksJ t| t|ks&J |   t| t|ks4J d S r  rN   r,   rU   r)   r   r   r   r    test_tokenize_random_state_numpy  s   r  moduler,   z	np.random)Zmarksc                 C  s   t | } | j}tt|}t|t|ksJ |  tt|}t|t|ks.J t|t|ks8J | d tt|}t|t|ksOJ t|t|ksYJ dS )zrandom.random() and other methods of the global random state do not compare as
    equal to themselves after a pickle roundtripr*   N)evalr,   pickler#   r$   r)   seed)r  rB   rC   r   r   r   r   r   test_tokenize_random_functions  s   
r  c                  C  s0   t dj } t dj }t| t|ksJ d S r  r  r   r   r   r   )test_tokenize_random_functions_with_state  s   r  c                  C  s4   t jdj} t jdj}t| t|ksJ d S r  r  r   r   r   r   /test_tokenize_random_functions_with_state_numpy  s   r  znot pac                  C  s(   t  } t  }t| t|ksJ d S r+   )parO   rQ   r)   r   r   r   r   &test_tokenize_pyarrow_datatypes_simple  s   r  c                  C  sH   t t  t  d} t t  t  d}t| t|ks"J d S )Nr   )r  structrP   r  rQ   Zint16r)   r   r   r   r   'test_tokenize_pyarrow_datatypes_complex  s   r  c                  C  s6   G dd d} t | ddgt | ddgksJ d S )Nc                   @  r=   )z3test_tokenize_opaque_object_with_buffers.<locals>.Cc                 S  s   t || _d S r+   )rN   r`   rl   r   r   r   r   r0   *  s   z<test_tokenize_opaque_object_with_buffers.<locals>.C.__init__Nr-  r   r   r   r   r   )  r@   r   rF   rG   rH   r   r   r   r   r   (test_tokenize_opaque_object_with_buffers&  s   (r  c                   @  s(   e Zd Zdd Zdd Ze Z ZZdS )
NumbaDummyc                 C  rh   )NFr   r.   r   r   r   __bool__3  r   zNumbaDummy.__bool__c                 O  s   dd }|S )Nc                 S  r   r+   r   r   r   r   r   wrapper7  r   z,NumbaDummy._dummy_decorator.<locals>.wrapperr   )r/   r%   r&   r  r   r   r   _dummy_decorator6  r   zNumbaDummy._dummy_decoratorN)r:   r;   r<   r  r  jit	vectorizeguvectorizer   r   r   r   r  2  s    r  Znopythonc                 C     | | S r+   r   r   r   r   r   	numba_jitA     r  
f8(f8, f8)c                 C  r  r+   r   r   r   r   r   numba_jit_with_signatureF  r  r  c                 C  r  r+   r   r   r   r   r   numba_vectorizeK  r  r  c                 C  r  r+   r   r   r   r   r   numba_vectorize_with_signatureP  r  r  f8,f8,f8[:]	(),()->()c                 C     | | |d< d S Nr   r   rl   r   outr   r   r   numba_guvectorizeU     r  z	not numbac                 C  s   | dddks	J t |  d S )NrF   rG   rH   r   r   r   r   r   test_tokenize_numbac  s   r  c                  C  s*   dd t D } t| tt| ksJ d S )Nc                 S  r   r   r   r   r   r   r   r   l  r   z4test_tokenize_numba_unique_token.<locals>.<listcomp>)all_numba_funcsr   r!   )r   r   r   r    test_tokenize_numba_unique_tokenj  s   r  c                  C  s   t jdddd } t jddddd }t jdddd	 }t jdddd
d }t dgddd }| ||||g}dd |D }t|tt|ksOJ d S )NTr  c                 S  r  r+   r   r   r   r   r   	local_jitr  r  z#test_numba_local.<locals>.local_jitr  c                 S  r  r+   r   r   r   r   r   local_jit_with_signaturev  r  z2test_numba_local.<locals>.local_jit_with_signaturec                 S  r  r+   r   r   r   r   r   local_vectorizez  r  z)test_numba_local.<locals>.local_vectorizec                 S  r  r+   r   r   r   r   r   local_vectorize_with_signature~  r  z8test_numba_local.<locals>.local_vectorize_with_signaturer  r  c                 S  r  r  r   r  r   r   r   local_guvectorize  r  z+test_numba_local.<locals>.local_guvectorizec                 S  r   r   r   r   r   r   r   r     r   z$test_numba_local.<locals>.<listcomp>)r   r  r  r  r   r!   )r  r  r  r  r  r   r   r   r   r   test_numba_localp  s$   






r  rB  rc   )
__future__r   rl  r   r   r  r   r  r,   r  r  r  enumr   r   r   r   typingr   r"   r   Ztlzr   r	   r
   r   r   r   r   r   Z	dask.corer   Z
dask.utilsr   Zdask.utils_testr   r   ddrN   r  r  r   r   Zfixturer   r)   rD   rL   markZskipifrR   rV   r\   r^   ra   rf   rp   rw   rz   r   r   r   r   r   r   r   r   r   r   r   r   r   getattrrJ   r   __str__r   r   r   Zparametrizer   r   r   Zxfailr   r   r   r   r   r   r   r   r  r  r  r  r  r  r  r  r%  r(  r*  r0  r4  r7  r9  r:  r@  rA  rF  rI  rK  rL  rO  rQ  rS  rZ  r\  ra  	dataclassrc  rh  rj  rn  rw  r  r  r  r  rd  r  r  filterwarningsr  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r   r   r   <module>   s   














	














K













	







	
	










*"












	







	


