
    tf                       d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlmZmZmZmZ d dlmZ d dlZd dlZd dlmZmZmZ d dlZd dlmZmZmZ d dlmZ d dl m!Z! d d	l"m#Z#  e#d
      Z$ e#d      Z% e#d      Z& e#d      Z' e#d      Z( e#d      Z) e#d      Z* ejV                  d      d        Z,d Z-d Z.d Z/ej`                  jc                  d      d        Z2ej`                  jc                  d      d        Z3ej`                  jc                  d      d        Z4ej`                  jc                  d      d        Z5ej`                  jc                  d      d        Z6ej`                  jc                  d      d        Z7ej`                  jc                  d      d        Z8ej`                  jc                  d      d        Z9ej`                  jc                  d      d         Z:ej`                  jc                  d      d!        Z;ej`                  jc                  d      d"        Z<ej`                  jc                  d      d#        Z=ej`                  jc                  d      d$        Z>ej`                  jc                  d      d%        Z?d& Z@d' ZAd( ZBd) ZCd* ZDd+aEd, ZF G d- d.      ZGeHeIj                  d/j                  eGj                   eG       j                  eGj                   eG       j                  eGj                  gZNej`                  j                  d0 eF             d1        ZPej`                  j                  d0eN      d2        ZQd3 ZRej`                  j                  d45      ej`                  j                  d6d7dg      d8               ZTd9 ZUdd:ZVdd;ZWd< ZXd= ZYd> ZZej`                  jc                  d?      d@        Z[ej`                  jc                  d?      dA        Z\ej`                  jc                  d?      dB        Z]ej`                  jc                  d?      dC        Z^ej`                  jc                  dD      dE        Z_ej`                  jc                  d?      dF        Z`ej`                  jc                  d?      dG        Zaej`                  jc                  d?      dH        ZbdI ZcdJ ZddK ZedL ZfdM ZgdN ZhdO ZidP Zj G dQ dR      Zk G dS dTek      ZldU ZmdV ZndW ZodX ZpdY ZqdZ Zrd[ Zsd\ Ztd] Zud^ Zvej`                  j                  d_eeeeg      d`        Zwej                   G da db             Zyej                   G dc dd             Zzej                   G de df             Z{ G dg dh      Z|di Z}dj Z~ej`                  j                  dkg dl      dm        Zej`                  jc                  d      dn        Zej`                  jc                  d      do        Zej`                  j                  dpd+ddqdrdsdtdtg dui deIeg      dv        Zdw Zej`                  jc                  d      ej`                  j                  dx      dy               Zej`                  jc                  dz      ej`                  j                  d{d|      d}               Zd~ Zej`                  j                  dkg d      d        Zd Zd Zd Zd Zd Zd Zej`                  jc                  d      d        Zej`                  j                  dd ej"                  dej`                  jc                  d            g      d        Zd Zej`                  jc                  d      d        Zej`                  jc                  d      d        Zej`                  jc                  d      d        Zej`                  jc                  d      d        Ze*s G d d      Z e       Z*e*j3                  d      d        Ze*j3                  dd      d        Ze*j9                  d      d        Ze*j9                  dd      d        Ze*j?                  dgd      d        ZeeeeegZej`                  jc                  d      ej`                  j                  d0e      d               Zej`                  jc                  d      d        Zej`                  jc                  d      d        Zy)    )annotationsN)EnumFlagIntEnumIntFlag)Union)composecurrypartial)TokenizationErrornormalize_tokentokenize)literal)tmpfile)import_or_nonez
dask.arrayzdask.dataframenumpyzscipy.sparsepyarrowpandasnumbaT)autousec               #    K   ddl m} m} t        j                  t
              5  | j                          ddd       t        j                  t
              5  |j                          ddd       d t        j                  t
              5  | j                          ddd       t        j                  t
              5  |j                          ddd       y# 1 sw Y   xY w# 1 sw Y   ~xY w# 1 sw Y   TxY w# 1 sw Y   yxY ww)zdTest that tokenize() and normalize_token() properly clean up context
    variables at all times
    r   _ensure_deterministic_seenN)	dask.baser   r   pytestraisesLookupErrorgetr   s     b/var/www/html/software/conda/envs/higlass/lib/python3.12/site-packages/dask/tests/test_tokenize.pycheck_contextvarsr!   #   s     
 7	{	# $!!#$	{	# 				{	# $!!#$	{	# 		 $ $ $ $ sc   "DC!DC$'%DC0!D>C<	DC!D$C-)D0C95D<DDc                    t         j                  j                  ddi      5  t        | i |}t        | i |}||k(  sJ t	        j
                  t	        j                  | |f            \  }}t	        j
                  t	        j                  | |f            \  }}t	        j
                  t	        j                  ||f            \  }}t        |i |}t        |i |}	||	k(  sJ 	 d d d        |S # 1 sw Y   S xY w)Ntokenize.ensure-deterministicT)daskconfigsetr   cloudpickleloadsdumps)
argskwargsbeforeafterargs2kwargs2args3kwargs3tok2tok3s
             r    check_tokenizer4   5   s    	94@	A 4*6* $)&) %**;+<+<dF^+LMw$**;+<+<dF^+LMw$**;+<+<eW=M+NOw*'**'*t||+2 M32 Ms   CC//C9c                 z   t        d       t        d      t        d      k(  sJ  G fdd              } t        j                  t              5  t        |        d d d         G d d      } |       }t        j                  t              5  t        |       d d d        y # 1 sw Y   HxY w# 1 sw Y   y xY w)N{   c                  $    e Zd Zd Z fdZd Zy)test_check_tokenize.<locals>.Ac                6    t        j                          | _        y N)randomtokselfs    r    __init__z'test_check_tokenize.<locals>.A.__init__Z   s    }}DH    c                    dfS N rC   )r>   As    r    
__reduce__z)test_check_tokenize.<locals>.A.__reduce__]   s    b5Lr@   c                    | j                   S r:   )r<   r=   s    r    __dask_tokenize__z0test_check_tokenize.<locals>.A.__dask_tokenize__`   s    88Or@   N)__name__
__module____qualname__r?   rE   rG   )rD   s   r    rD   r8   Y   s    	'		r@   rD   c                      e Zd Zd Zy)test_check_tokenize.<locals>.Bc                *    t        j                          S r:   )r;   r=   s    r    rG   z0test_check_tokenize.<locals>.B.__dask_tokenize__i   s    ==?"r@   NrH   rI   rJ   rG   rC   r@   r    BrL   h   s    	#r@   rO   )r4   r   r   r   AssertionError)arO   brD   s      @r    test_check_tokenizerS   R   s    3C=N3////  	
A	~	& q# # 	
A	~	& q   s   B%B1%B.1B:c                 >    d} t        t        |       t              sJ y )N         )
isinstancer   strrQ   s    r    test_tokenizer\   q   s    Ahqk3'''r@   znot npc                 .   t        t        j                  d            t        d      k7  sJ t        t        j                  d            t        t        j                  d            k7  sJ t        t        j                  d            t        t        j	                  d            k7  sJ t        t        j                  d            t        d      k7  sJ t        t        j                  d            t        t        j                  d            k7  sJ y )NrV   1)r4   npint64int32uint32float64rC   r@   r    test_tokenize_scalarrd   v   s    "((1+&.*;;;;"((1+&.!*EEEE"((1+&.1*FFFF"((1+&.*===="((1+&.A*GGGGr@   c                     t        t        j                  j                  d      j	                  d            t        t        j                  j                  d      j	                  d            k(  sJ y )N    )r4   r_   r;   RandomStaterandom_samplerC   r@   r    .test_tokenize_numpy_array_consistent_on_valuesrj      s[    
		d#11$7			--d3AA$G	HI I Ir@   c                 t    t        t        j                  j                  d      j                  d             y )N   i2dtype)r4   r_   r;   astyperC   r@   r    /test_tokenize_numpy_array_supports_uneven_sizesrq      s'    299##A&--D-9:r@   c                 `    t        t        j                  j                  d      d d d          y )N   rW   )r4   r_   r;   rC   r@   r    'test_tokenize_discontiguous_numpy_arrayrt      s"    299##A&ss+,r@   c                 F    t        t        j                  dgd             y )Nz2000-01-01T12:00:00zM8[ns]rn   r4   r_   arrayrC   r@   r    test_tokenize_numpy_datetimerx      s    2882388DEr@   c                 (   t        t        j                  dd            t        t        j                  dd            k(  sJ t        t        j                  dgddg      d         t        t        j                  dgddg      d         k(  sJ y )N      ?f8rn   rV   rW   )rQ   i4)rR   i8r   rv   rC   r@   r    test_tokenize_numpy_scalarr      s    "((3d(34
D!9    
&+{!;<Q?	&+{1KLQO	PQ Q Qr@   c                     t         j                  dd i      5  t        t         j                  d            t        t         j                  d            k7  sJ 	 d d d        y # 1 sw Y   y xY w)Nallc                     yNfoorC   xs    r    <lambda>z7test_tokenize_numpy_scalar_string_rep.<locals>.<lambda>   s    r@   )	formatterrV   rW   )r_   printoptionsr4   rw   rC   r@   r    %test_tokenize_numpy_scalar_string_repr      sX     
E?#;	< Jbhhqk*nRXXa[.IIIIJ J Js   A A$$A-c                    t         j                  g dt              } t        |       t        |       k(  sJ t        t         j                  g dt                    t        t         j                  g dt                    k(  sJ t        t         j                  g dt                    t        t         j                  g dt                    k(  sJ  G d d      }t         j                  dd  |       gt              }t        |       y )N)rQ   aaaaarn   )rQ   Nr   ))rV   rQ   )rV   N)rV   r   c                      e Zd Zy)Ctest_tokenize_numpy_array_on_object_dtype.<locals>.NeedsCloudPickleNrH   rI   rJ   rC   r@   r    NeedsCloudPickler          r@   r   rQ   )r_   rw   objectr4   )rQ   r   r   s      r    )test_tokenize_numpy_array_on_object_dtyper      s    
#62A!q 1111"((#5V(DE
#62J    
2&A	!B&Q	RS S S  	#t-/0?A1r@   c                     t         j                  g       } | j                  sJ t        |       t        |       k(  sJ t         j                  g t         j	                               }t        |       t        |      k7  sJ y )Nrn   )r_   rw   stridesr4   r`   )arrarr2s     r    test_empty_numpy_arrayr      sc    
((2,C;;;#."555588Bbhhj8)D#."6666r@   c                   t        | j                  d            }t        |d      5 }|j                  d       d d d        t        |d      5 }t        j                  |t        j                  ddd      }t        j                  |t        j                  ddd      }t        j                  |t        j                  ddd      }t        |      t        |      k(  sJ t        |      t        |      k(  sJ t        |      t        |      k7  sJ t        |d	d
       t        |d	d
       k(  sJ t        |d	d
       t        |d	d
       k(  sJ t        |d	d       t        |d	d       k(  sJ t        |d	d       t        |d	d       k7  sJ t        |d	d       t        |d	d       k7  sJ |d	d
 }|d	d
 }t        |      t        |      k7  sJ 	 d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)N	demo_datawbs
   ashekwichtrbrr      )ro   modeoffsetshaperV   rW   rX   )rZ   joinopenwriter_   memmapuint8r4   )tmpdirfnfmmap1mmap2mmap3sub1sub2s           r    !test_tokenize_numpy_memmap_offsetr      s    
V[[%	&B	b$ 1	 
b$ <1		!288#aq	I		!288#aq	I		!288#aq	Ie$u(====e$u(====e$u(====eAbk*nU1R[.IIIIeAbk*nU1R[.IIIIeAaj)^E!AJ-GGGGeAaj)^E!AJ-GGGGeAaj)^E!AJ-GGGGQr{Qr{d#~d';;;;< < < <s   GFG&G#&G/c                 :   t        d      5 } t        j                  d      }t        j                  | |       t	        t        j                  | d            }d d d        t        d      5 } t        j                  d      }t        j                  | |       t	        t        j                  | d            }d d d        t	              t	              k(  sJ k7  sJ t        d      5 } t        j                  j                  d      }t        j                  | |       t        j                  | d      }t        j                  | d      }t	        |dd d f         }t	        |dd d f         }	t	        |dd	d d f         }
t	        |d d df         }t        ||	|
|h      d
k(  sJ t	        |      t	        |      k(  sJ t	        |dd d f         t	        |dd d f         k(  sJ 	 d d d        y # 1 sw Y   xY w# 1 sw Y   OxY w# 1 sw Y   y xY w)N.npyr   r   	mmap_mode)
   r   )sizer   rV   rX      )	r   r_   arangesaver4   loadr;   normallen)r   x1yx2zr   mmmm2rQ   rR   cds               r    test_tokenize_numpy_memmapr      s   	 7BYYq\
B27727567
 
 7BYYq\
B27727567
 "!33336M6	 EBII(+
AWWR3W'ggbCg(2ad8$2ad8$2ac1f:&2ad8$Aq!Q< A%%%b!^C%8888bAh'>#ad)+DDDDE E7 7
7 7E Es&   AG7+AH*DH7HHHc                 r   t        d      5 } t        d      5 }t        j                  d      }t        j                  | |       t        j                  ||       t        j	                  | d      }||z   }t        |      t        |      k(  sJ 	 d d d        d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)Nr   r   r   r   )r   r_   r   r   r   r4   )fn1fn2r   rQ   rR   s        r    &test_tokenize_numpy_memmap_no_filenamer      s     
 6C 6CIIaL
Q
QGGC3G'Ea N1$55556 6 6 6 6 6s#   B-A7B!B-!B*	&B--B6c                    t        t        j                        t        t        j                        k7  sJ t        j                  } t        j                  }t	        | t        j
                        sJ t	        |t        j
                        sJ t        |       t        |      k7  sJ t        j
                  j                  d dd      }t        j
                  j                  d dd      }t        j
                  j                  d dd      }t        |      t        |      k7  sJ t        |      t        |      k7  sJ y )Nc                    | dz   S NrV   rC   r   s    r    r   z+test_tokenize_numpy_ufunc.<locals>.<lambda>  s
    A r@   rV   c                    | dz   S r   rC   r   s    r    r   z+test_tokenize_numpy_ufunc.<locals>.<lambda>  
    Q r@   c                    | dz   S NrW   rC   r   s    r    r   z+test_tokenize_numpy_ufunc.<locals>.<lambda>  r   r@   )r4   r_   sincosrY   ufuncda
frompyfunc)np_ufunc	np_ufunc2incinc2inc3s        r    test_tokenize_numpy_ufuncr     s    "&&!^BFF%;;;;vvHIh)))i***(#~i'@@@@ ((

oq!
4C8815D8815D#."6666#."6666r@   c                    t         j                  d dd      } t        j                  j	                  ddi      5  t        |       t        |       k7  sJ t        |       t        |       k7  sJ 	 d d d        t        j                  j	                  ddi      5  t        j                  t        d      5  t        |        d d d        d d d        t        | d      t        | d      k7  sJ t        j                  t        d	      5  t        | d       d d d        y # 1 sw Y   xY w# 1 sw Y   lxY w# 1 sw Y   pxY w# 1 sw Y   y xY w)
Nc                    | dz   S r   rC   r   s    r    r   z;test_normalize_numpy_ufunc_unserializable.<locals>.<lambda>  s
    !a% r@   rV   r#   FTz,Cannot tokenize.*dask\.array\.ufunc.*insteadmatchensure_deterministiczCannot tokenize)
r_   r   r$   r%   r&   r   r   r   r   r   )r   s    r    )test_normalize_numpy_ufunc_unserializabler     s)    --A
.C	95A	B <}---s#s';;;;	< 
94@	A ]]%T
 	 SM	 Ce4%9    
(0A	B 1401 1!< <	 	 1 1s;   3DD63D*?D6ED'*D3	/D66D?Ec                     G d d      }  |        }t         j                  j                  ddi      5  t        |      t        |      k7  sJ t	        |      t	        |      k7  sJ 	 d d d        t         j                  j                  ddi      5  t        j                  t        d      5  t        |       d d d        d d d        t        |d      t        |d      k7  sJ t        j                  t        d      5  t        |d       d d d        y # 1 sw Y   xY w# 1 sw Y   lxY w# 1 sw Y   pxY w# 1 sw Y   y xY w)	Nc                      e Zd Zd Zy)/test_normalize_object_unserializable.<locals>.Cc                    J r:   rC   r=   s    r    rE   z:test_normalize_object_unserializable.<locals>.C.__reduce__1      5r@   NrH   rI   rJ   rE   rC   r@   r    Cr   0      	r@   r   r#   FTz"cannot be deterministically hashedr   r   )r$   r%   r&   r   r   r   r   r   )r   r   s     r    $test_normalize_object_unserializabler   /  s(     	
A	95A	B 8{hqk)))q!_Q%7777	8 
94@	A ]]%I
 	 QK	 AE2h	7    
(0T	U /./ /!8 8	 	 / /s;   3DD/,D#8D/ D;D #D,	(D//D8;Ec                     t        t        t        t              } t        t        t        t              }t        t        t        d      }t	        |       t	        |      k(  sJ t	        |       t	        |      k7  sJ y )N)r   r   )r   f3f2f1r4   )r   ghs      r    1test_tokenize_partial_func_args_kwargs_consistentr   J  s\    B"AB"AB!A!q 1111!q 1111r@   c            
         dddt        ddd      t        j                  d      t        j                  ddd	      t        j                  d
      fD ]  } t        |       | u rJ  y )NrV   g?r^   rW   rX   z1.1        z
/this/that)slicedecimalDecimaldatetimedatepathlibPurePathr   )is    r    test_normalize_baser   R  sf    	aAdAr"& 	' q!Q&&&	'r@   c            
        t         j                  j                  ddi      5  t               } t	        |       t	        |       k(  sJ t	        t                     t	        t                     k7  sJ t        t        d      D ch c]  }t	        t                      c}      dk(  sJ t        |       t        |       k(  sJ 	 d d d        t         j                  j                  ddi      5  t        j                  t        d      5  t	                d d d        t        j                  t        d      5  t                d d d        d d d        t	         d      t	        | d      k(  sJ t        j                  t        d      5  t	        | d       d d d        y c c}w # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   }xY w# 1 sw Y   xY w# 1 sw Y   y xY w)Nr#   Fd   Tdeterministicr   r   )r$   r%   r&   r   r   r   ranger   r   r   r   )o_s     r    test_tokenize_objectr  _  s   	95A	B 8H{hqk)))!Xfh%7777
 c
;1HVX&;<CCC q!_Q%77778  
94@	A ]],OD 	QK	]],OD 	A	 AE2h	7    
(	@ /./ /! <8 8"	 		 	 / /sf   AF%F 
%F%*G	F1#G	5F=G		G F%%F.1F:	6G	=G	G		GGc                      G d d      } t         |        d      t         |        d      k7  sJ t        j                  t              5  t         |               ddd       y# 1 sw Y   yxY w)zdTest that the ensure_deterministic override is not lost if tokenize() is
    called recursively
    c                      e Zd Zd Zy)/nested_tokenize_ensure_deterministic.<locals>.Cc                (    t        t                     S r:   )r   r   r=   s    r    rG   zAnested_tokenize_ensure_deterministic.<locals>.C.__dask_tokenize__  s    FH%%r@   NrN   rC   r@   r    r   r
    s    	&r@   r   Fr   N)r   r   r   r   r   s    r    $nested_tokenize_ensure_deterministicr  ~  sd    
& & ACe4	%9    
(	)   s   A%%A.rV   c                    d d d d g} | d d \  d }d }| ||gz  } dfd	}| j                  |       d
 }| j                  |       fd}| j                  |       fd}| j                  |       d }| j                  |       d }| j                  |       fd}| j                  |       fd}| j                  |       | S )Nc                    | S r:   rC   r   s    r    r   z"_local_functions.<locals>.<lambda>      ! r@   c                    | dz   S r   rC   r   s    r    r   z"_local_functions.<locals>.<lambda>  
    !a% r@   c                    | S r:   rC   r   s    r    r   z"_local_functions.<locals>.<lambda>  r  r@   c                    | dz   S r   rC   r  s    r    r   z"_local_functions.<locals>.<lambda>  r  r@   rW   c                    | S r:   rC   r   s    r    funcz_local_functions.<locals>.func      r@   c                    | S r:   rC   r   s    r    r   z_local_functions.<locals>.f2  r  r@   rV   c                     dz          S r   rC   )rQ   local_scopes   r    r  z_local_functions.<locals>.func  s    q~r@   c                      t         dz  a t         S r   )_GLOBALrC   r@   r    r  z_local_functions.<locals>.func  s    1r@   c                     ||       S r:   rC   r   r   s     r    r  z_local_functions.<locals>.func      tr@   c                     ||       S r:   rC   r  s     r    r  z_local_functions.<locals>.func  r   r@   c                    d } ||       S )Nc                    | dz   S r   rC   r   s    r    r   z0_local_functions.<locals>.func.<locals>.<lambda>  
    a!e r@   rC   r  s     r    r  z_local_functions.<locals>.func      tr@   c                    d } ||       S )Nc                    | dz   S )NrX   rC   r   s    r    r   z0_local_functions.<locals>.func.<locals>.<lambda>  r$  r@   rC   r  s     r    r  z_local_functions.<locals>.func  r%  r@   c                    } ||       S r:   rC   )r   r   rQ   s     r    r  z_local_functions.<locals>.func      tr@   c                    } ||       S r:   rC   )r   r   rR   s     r    r  z_local_functions.<locals>.func  r)  r@   )append)	all_funcsr  r   rQ   rR   r  s      @@@r    _local_functionsr-    s    	I Ra=DAq $IK
 T
 T
   T  T T T T Tr@   c                  "    e Zd Zd Zed        Zy)WithClassMethodc                     y r:   rC   r=   s    r    r   zWithClassMethod.f      r@   c                     y r:   rC   clss    r    r   zWithClassMethod.g      r@   N)rH   rI   rJ   r   classmethodr   rC   r@   r    r/  r/    s      r@   r/  r   r  c                    t        |        y r:   r4   r  s    r    test_tokenize_local_functionsr:    
    4r@   c                    t        |        y r:   r8  r9  s    r    test_tokenize_special_callablesr=    r;  r@   c                     t               t        z   } | D cg c]  }t        |       }}t        t	        |            t        |      k(  sJ y c c}w r:   )r-  _special_callablesr4   r   r&   )r,  r  tokenss      r    $test_tokenize_functions_unique_tokenrA    sH     "%77I/89tnT"9F9s6{s6{*** :s   Az3https://github.com/cloudpipe/cloudpickle/issues/453)reasoninstanceFc                V      fd}t         |             t         |             k(  sJ y )Nc                 .     G d d      } r |        S | S )Nc                      e Zd Zy)Itest_tokenize_local_classes_from_different_contexts.<locals>.f.<locals>.CNr   rC   r@   r    r   rG    s    r@   r   rC   )r   rC  s    r    r   z>test_tokenize_local_classes_from_different_contexts.<locals>.f  s    	 	 qs%A%r@   r8  )rC  r   s   ` r    3test_tokenize_local_classes_from_different_contextsrH    s&    & !#."5555r@   c                 P    d } t         |              t         |              k(  sJ y )Nc                     d } | S )Nc                      y)Nr6   rC   rC   r@   r    r   zKtest_tokenize_local_functions_from_different_contexts.<locals>.f.<locals>.g      r@   rC   )r   s    r    r   z@test_tokenize_local_functions_from_different_contexts.<locals>.f  s    	 r@   r8  )r   s    r    5test_tokenize_local_functions_from_different_contextsrM    s%     !#."5555r@   c                     y r:   rC   rQ   rR   r   s      r    r   r         r@   c                     y r:   rC   rO  s      r    r   r     rP  r@   c                     y r:   rC   r[   s    r    r   r     rP  r@   c                 F    t        t              t        t              k7  sJ y r:   )r4   r   r   rC   r@   r    test_tokenize_callablerT  #  s    "!3333r@   c                 .   t        t        t        d            t        t        t        d            k7  sJ t        t        t        d            t        t        t        d            k7  sJ t        t	        t        t
                    t        t	        t        t                    k7  sJ t        t        t                    t        t        t                    k7  sJ t        t        t        d            t        t        t        d            k7  sJ y )NrW   )rR   rX   rV   )r4   r   r   r   r	   r   r
   rC   r@   r    !test_tokenize_composite_functionsrV  '  s    '"*+~gbA>N/OOOO'"*+~gbA>N/OOOO'"b/*nWR_.MMMM%)$uRy(AAAA%a.)^E"N-KKKKr@   znot pdc                 V   t         j                  g dg ddg d      } t         j                  g dg ddg d      }t        |       t        |      k(  sJ d|j                  _        t        |       t        |      k7  sJ t         j                  g dg dd      } t         j                  g dg dd      }| j
                  j                  d      | d<   t        |       t        |      k7  sJ | j
                  j                  d      |d<   t        |       t        |      k(  sJ y )	NrU   )4asdNr   r   indexr   )rQ   rR   rQ   categoryr   )pd	DataFramer4   r\  namer   rp   rQ   rR   s     r    test_tokenize_pandasrb  /  s    
9+=>iPA
9+=>iPA!q 1111AGGL!q 1111
9?;<A
9?;<ASSZZ
#AcF!q 1111SSZZ
#AcF!q 1111r@   c                 Z    t         j                  g dg ddg d      } t        |        y )NrU   )rX  u   asdN)u   xu   yr[  r^  r_  r4   dfs    r    $test_tokenize_pandas_invalid_unicoderg  @  s,     
*BC9 
 
B 2r@   c                     t         j                  dj                         g ddddj                         d gig d      } t        |        y )N   örU   r[  )r^  r_  encoder4   re  s    r    (test_tokenize_pandas_mixed_unicode_bytesrk  I  sA    			4$t)DE 
 
B 2r@   c                 l     G d d      } t         j                  ddd  |        gi      }t        |       y )Nc                      e Zd Zy):test_tokenize_pandas_cloudpickle.<locals>.NeedsCloudPickleNr   rC   r@   r    r   rn  T  s    r@   r   r   r   rd  )r   rf  s     r     test_tokenize_pandas_cloudpicklero  R  s4      
sUD*:*<=>	?B2r@   znot ddc                 n   t         j                  g dd      t         j                  dgd      t         j                  g dd      t         j                  t         j                  d      gd      t         j                  t         j                  dd	
      gd      t         j                  ddgt         j                  j                  j                  g dd            g} | j                  t         j                  g dd      t         j                  g dd      g       | D ]  }t        |        y )N)rV   r   NInt64rn   2000z	Period[D])rV   r   r   zSparse[int]zdatetime64[ns]CET)tzzdatetime64[ns, CET]rQ   rR   rO  F)ordered)rQ   rR   Nstring)TFNboolean)r^  rw   	TimestampapitypesCategoricalDtypeextendr4   )arraysr   s     r    $test_tokenize_pandas_extension_arrayr~  \  s     	W-
&-
-0
",,v&'/?@
",,v%,019NO
#J&&,,///O 	 	

F MMHH%XH6HH(	H:	
  sr@   c                 6    t        t        j                         y r:   )r4   r^  NArC   r@   r    test_tokenize_nar  u  s    255r@   c            	        t         j                  j                  d      t         j                  j                  d      t         j                  j	                  d      t         j                  j                  d      t         j                  d      t         j                  d      t         j                  d      fD ]  } t        |         y )NrV   rW   )yearsrl   )monthsr   days)r^  offsetsSecond
MonthBeginDayBQuarterEnd
DateOffsetr4   )r   s    r    test_tokenize_offsetr  z  s     	

!


a 


q


q!
A
Q
2 	 	v	r@   c                     t         j                  ddg      } t        |        t         j                  j	                  ddgddgg      } t        |        y )NrQ   rR   r   rV   )r^  Indexr4   
MultiIndexfrom_product)idxs    r    test_tokenize_pandas_indexr    sE    
((C:
C3
--
$
$sCj1a&%9
:C3r@   c                    t        dd       t        d      t        dd      k7  sJ t        dd      t        dd      k7  sJ t        dd      t        dd      k7  sJ t        dd      t        dddi      k7  sJ y )	Nr   rV   r   rW   r  bar)r   r   r8  rC   r@   r    test_tokenize_kwargsr    s~    1!qA 6666!q!^A%;;;;!q!^A%;;;;!'>!eU^+LLLLr@   c                 b     G d d      } t         | d            t         | d            k7  sJ y )Nc                      e Zd Zd Zd Zy)$test_tokenize_same_repr.<locals>.Fooc                    || _         y r:   r   r>   r   s     r    r?   z-test_tokenize_same_repr.<locals>.Foo.__init__  	    DFr@   c                     y)Nza foorC   r=   s    r    __repr__z-test_tokenize_same_repr.<locals>.Foo.__repr__  s    r@   N)rH   rI   rJ   r?   r  rC   r@   r    Foor    s    		r@   r  rV   rW   r8  r  s    r    test_tokenize_same_reprr    s.      #a&!^CF%;;;;r@   c                 b     G d d      } t         | d            t         | d            k7  sJ y )Nc                      e Zd ZdZd Zy)"test_tokenize_slotted.<locals>.Foor   c                    || _         y r:   r   r  s     r    r?   z+test_tokenize_slotted.<locals>.Foo.__init__  r  r@   NrH   rI   rJ   	__slots__r?   rC   r@   r    r  r    s    		r@   r  rV   rW   r8  r  s    r    test_tokenize_slottedr    s.      #a&!^CF%;;;;r@   c                      G d d      } t         | d            t         | d            k7  sJ t         |               y )Nc                      e Zd ZdZddZy)+test_tokenize_slotted_no_value.<locals>.FoorZ  Nc                *    ||| _         ||| _        y y r:   rZ  r>   r   r   s      r    r?   z4test_tokenize_slotted_no_value.<locals>.Foo.__init__  s    }} r@   )NNr  rC   r@   r    r  r    s    		r@   r  rV   r   r  r8  r  s    r    test_tokenize_slotted_no_valuer    s8      #(#~cAh'????35r@   c            	     2    G d d      }  G d d|       } |dd      j                   ddik(  sJ t         |dd            t         |dd            t         |dd             t         |dd            g}t        t        |            t        |      k(  sJ y )	Nc                      e Zd ZdZy))test_tokenize_slots_and_dict.<locals>.Foor   N)rH   rI   rJ   r  rC   r@   r    r  r    s    	r@   r  c                      e Zd Zd Zy))test_tokenize_slots_and_dict.<locals>.Barc                &    || _         ||| _        y y r:   rZ  r  s      r    r?   z2test_tokenize_slots_and_dict.<locals>.Bar.__init__  s    DF} r@   NrH   rI   rJ   r?   rC   r@   r    Barr    s    	r@   r  rV   rW   r   rX   )__dict__r4   r   r&   )r  r  r@  s      r    test_tokenize_slots_and_dictr    s     c  q!9#q))) 	s1ay!s1ay!s1d|$s1ay!	F s6{s6{***r@   c                 N    G d d      }  | d       | d      }}t        |      t        |      k7  sJ t        |j                        t        |j                        k7  sJ t        |      }t        j                  | d        t        |      }||k7  sJ t        j                  | = y )Nc                      e Zd Zd Zd Zd Zy)!test_tokenize_method.<locals>.Fooc                    || _         y r:   r   r  s     r    r?   z*test_tokenize_method.<locals>.Foo.__init__  r  r@   c                    | j                   S r:   r   r=   s    r    rG   z3test_tokenize_method.<locals>.Foo.__dask_tokenize__      66Mr@   c                     y)NzHello worldrC   r=   s    r    helloz'test_tokenize_method.<locals>.Foo.hello  s     r@   N)rH   rI   rJ   r?   rG   r  rC   r@   r    r  r    s    			!r@   r  rV   rW   c                     | j                   dz   S r   r   r=   s    r    r   z&test_tokenize_method.<locals>.<lambda>  s    tvvz r@   )r4   r  r   register_lookup)r  rQ   rR   r,   r-   s        r    test_tokenize_methodr    s    ! ! q63q6qA!q 1111!''"nQWW&==== AFS"9:1EU??$r@   c                      G d d      }  G d d|       } | d       | d       |d      }}}t        |      t        |      k7  sJ t        |      t        |      k7  sJ y )Nc                      e Zd Zd Zd Zy)'test_tokenize_callable_class.<locals>.Cc                    || _         y r:   r   r  s     r    r?   z0test_tokenize_callable_class.<locals>.C.__init__  r  r@   c                    | j                   S r:   r   r=   s    r    __call__z0test_tokenize_callable_class.<locals>.C.__call__  r  r@   N)rH   rI   rJ   r?   r  rC   r@   r    r   r    s    		r@   r   c                      e Zd Zy)'test_tokenize_callable_class.<locals>.DNr   rC   r@   r    Dr    r   r@   r  rV   rW   r8  )r   r  rQ   rR   r   s        r    test_tokenize_callable_classr    sc     A  dAaD!A$!qA!q 1111!q 1111r@   c                      G d d      } t         | dd            t         | dd            k(  sJ t         | dd            t         | dd            k7  sJ y)z(Always use ___dask_tokenize__ if presentc                      e Zd Zd Zd Zd Zy)<test_tokenize_callable_class_with_tokenize_method.<locals>.Cc                     || _         || _        y r:   rZ  r  s      r    r?   zEtest_tokenize_callable_class_with_tokenize_method.<locals>.C.__init__  s    DFDFr@   c                    | j                   S r:   r   r=   s    r    rG   zNtest_tokenize_callable_class_with_tokenize_method.<locals>.C.__dask_tokenize__  r  r@   c                     y r:   rC   r=   s    r    r  zEtest_tokenize_callable_class_with_tokenize_method.<locals>.C.__call__  s    r@   N)rH   rI   rJ   r?   rG   r  rC   r@   r    r   r    s    			r@   r   rV   rW   rX   Nr8  r  s    r    1test_tokenize_callable_class_with_tokenize_methodr    sV    	 	 !Aq'"nQq!W&====!Aq'"nQq!W&====r@   c                  >    e Zd Zd Zd Zd Zed        Zed        Z	y)HasStaticMethodsc                    || _         y r:   r   r  s     r    r?   zHasStaticMethods.__init__  s	    r@   c                B    t        t        |             | j                  fS r:   )r   typer   r=   s    r    rG   z"HasStaticMethods.__dask_tokenize__  s    tDz*DFF22r@   c                     y r:   rC   r=   s    r    normal_methodzHasStaticMethods.normal_method  r1  r@   c                      y r:   rC   rC   r@   r    static_methodzHasStaticMethods.static_method  r5  r@   c                     y r:   rC   r3  s    r    class_methodzHasStaticMethods.class_method  r5  r@   N)
rH   rI   rJ   r?   rG   r  staticmethodr  r6  r  rC   r@   r    r  r    s9    3    r@   r  c                      e Zd Zy)HasStaticMethods2Nr   rC   r@   r    r  r    s    r@   r  c                 b   t        d      t        d      t        d      }}} t        | j                        t        |j                        k7  sJ t        | j                        t        |j                        k7  sJ t        | j                        t        |j                        k(  sJ t        | j                        t        |j                        k(  sJ t        | j
                        t        |j
                        k(  sJ t        | j
                        t        |j
                        k7  sJ y )NrV   rW   )r  r  r4   r  r  r  rO  s      r    test_staticmethodsr  "  s    q!#3A#68I!8L!qA!//*nQ__.MMMM!//*nQ__.MMMM!//*nQ__.MMMM!//*nQ__.MMMM!..)^ANN-KKKK!..)^ANN-KKKKr@   c            	         t        dg      t        dg      k7  sJ t        dg      t        d      k7  sJ t        dg      t        dg      k(  sJ d} | | | dgg}t        |      ddddgfdddddd	gfgfgfk(  sJ y )
NrV   rW   rV   r|   )rW   rX   listtuple)__seenr   rX   )r4   r   rZ  s     r    test_tokenize_sequencesr  .  s    1#.!"55551#."66661#.!"5555 	A	
A6{A1q!fmg1v%678	
"   r@   c                     g d}  G d d      } ||        ||       }}t        | ||        |j                  sJ t        |      t        |      k(  sJ y)zuTest that calling tokenize() recursively doesn't alter the output due to
    memoization of already-seen objects
    rU   c                      e Zd Zd Zd Zy)$test_nested_tokenize_seen.<locals>.Cc                     || _         d | _        y r:   )r   r<   r  s     r    r?   z-test_nested_tokenize_seen.<locals>.C.__init__H  s    DFDHr@   c                f    | j                   st        | j                        | _         | j                   S r:   )r<   r   r   r=   s    r    rG   z6test_nested_tokenize_seen.<locals>.C.__dask_tokenize__L  s"    88#DFF+88Or@   N)rH   rI   rJ   r?   rG   rC   r@   r    r   r  G  s    		r@   r   N)r4   r<   )r  r   c1c2s       r    test_nested_tokenize_seenr  A  sU     	A  qT1Q4B1b!66M6"!3333r@   c                     t        ddd      t        ddd      k(  sJ t        ddd      t        ddd      k7  sJ t        ddd      t        ddd      k7  sJ y )NrV   r   )r   rV   rV   r   )r   rW   rW   r8  rC   r@   r    test_tokenize_dictr  W  sh    c*+~#A>N/OOOOc*+~A#>N/OOOOc*+~A#>N/OOOOr@   c                 >    t        h d      t        h d      k(  sJ y )N>   r  r   rV   rW   r8  rC   r@   r    test_tokenize_setr  ^  s$    /0N5   r@   c                     ddl m}   | ddg      } | ddg      } | ddg      }t        |      t        |      k(  sJ t        |      t        |      k7  sJ y )Nr   )OrderedDict)rQ   rV   )rR   rW   )collectionsr  r4   )r  rQ   rR   r   s       r    test_tokenize_ordered_dictr  d  sb    'Xx()AXx()AXx()A!q 1111!q 1111r@   c                 H     G d d      } t         |         |        d       y )Nc                      e Zd Zd Zd Zy)7test_tokenize_dict_doesnt_call_str_on_values.<locals>.Cc                     y)Nr   rC   r=   s    r    rG   zItest_tokenize_dict_doesnt_call_str_on_values.<locals>.C.__dask_tokenize__q  rL  r@   c                    J r:   rC   r=   s    r    r  z@test_tokenize_dict_doesnt_call_str_on_values.<locals>.C.__repr__t  r   r@   N)rH   rI   rJ   rG   r  rC   r@   r    r   r  p  s    		r@   r   )rV   2r8  r  s    r    ,test_tokenize_dict_doesnt_call_str_on_valuesr  o  s      qs%&r@   c                 ~    d} | | d}| | d}dt        t        |            v sJ t        |      t        |      k(  sJ y)zWhen sequence values are repeated, the 2nd+ entry is tokenized as (__seen, 0).
    This makes it important to ensure that dicts are sorted *before* you call
    normalize_token() on their elements.
    rU   r|   )rW   rV   r  N)rZ   r   r4   )vd1d2s      r    (test_tokenize_sorts_dict_before_seen_mapr  z  sK    
 	A1B1Bs?2.////"!3333r@   c                     d} t        d      D ch c]  }|| f }}t        t        d            D ch c]  }|| f }}dt        t        |            v sJ t	        |      t	        |      k(  sJ yc c}w c c}w )zSame as test_tokenize_sorts_dict_before_seen_map, but for sets.

    Note that this test is only meaningful if set insertion order impacts iteration
    order, which is an implementation detail of the Python interpreter.
    rU   r  r  N)r  reversedrZ   r   r4   )r  r   s1s2s       r    'test_tokenize_sorts_set_before_seen_mapr    s|     	A*	%Q1a&	%B	%"5:.	/Q1a&	/B	/s?2.////"!3333 
&	/s
   A2A7c                    t        t        j                  d            t        t        j                  d            k(  sJ t        t        j                  d            t        t        j                  d            k7  sJ y )NrV   r  rW   )r4   r   	timedeltarC   r@   r    test_tokenize_timedeltar    sr    (,,!45":    (,,!45":   r@   	enum_typec                     G d d|       }t        |j                        t        |j                        k(  sJ t        |j                        t        |j                        k7  sJ y )Nc                      e Zd ZdZdZy)!test_tokenize_enum.<locals>.ColorrV   rW   N)rH   rI   rJ   REDBLUErC   r@   r    Colorr    s    r@   r  )r4   r  r  )r  r  s     r    test_tokenize_enumr    sP    	  %))$uyy(AAAA%))$uzz(BBBBr@   c                      e Zd ZU ded<   y)
ADataClassintrQ   NrH   rI   rJ   __annotations__rC   r@   r    r  r    s    
Fr@   r  c                      e Zd ZU ded<   y)
BDataClassfloatrQ   Nr  rC   r@   r    r  r    s    Hr@   r  c                  >    e Zd ZU  ej                  d      Zded<   y)NoValueDataClassF)initr  rQ   N)rH   rI   rJ   dataclassesfieldrQ   r  rC   r@   r    r"  r"    s    [E*As*r@   r"  c                      e Zd ZddZy)GlobalClassc                    || _         y r:   )val)r>   r)  s     r    r?   zGlobalClass.__init__  s	    r@   N)returnNoner  rC   r@   r    r'  r'    s    r@   r'  c                 j    G d d      }  G fdd       G d d      } G d dt               }t        t        d	             t        t        d	            t        t        d
            k7  sJ t         |               t         |d	             t         |             t                      k7  sJ y )Nc                      e Zd ZdZy)%test_local_objects.<locals>.LocalTyper  N)rH   rI   rJ   r   rC   r@   r    	LocalTyper.    s    r@   r/  c                      e Zd Z fdZy)*test_local_objects.<locals>.LocalReduciblec                    dfS rB   rC   )r>   LocalReducibles    r    rE   z5test_local_objects.<locals>.LocalReducible.__reduce__  s    !2%%r@   Nr   )r3  s   r    r3  r1    s    	&r@   r3  c                      e Zd Zd Zy)-test_local_objects.<locals>.LocalDaskTokenizec                     yr   rC   r=   s    r    rG   z?test_local_objects.<locals>.LocalDaskTokenize.__dask_tokenize__      r@   NrN   rC   r@   r    LocalDaskTokenizer5    r   r@   r8  c                      e Zd Zy)&test_local_objects.<locals>.LocalChildNr   rC   r@   r    
LocalChildr:    r   r@   r;  rV   rW   )r'  r4   )r/  r8  r;  r3  s      @r    test_local_objectsr<    s     & & [  ;q>"+a.)^KN-KKKK9;:a=!+-..AQ2RRRRr@   c                    t        d      } t        d      }t        |        t        |       t        |      k7  sJ t        d      }t        t               t        t              k7  sJ t        |       t        |      k7  sJ  G d dt               }t        j                  |      sJ t        t               t        |      k7  sJ t         |d            t        |       k7  sJ t        j
                  ddt        t        t        f   fg      }t        |       t         |d            k7  sJ t               }t        |       y )NrV   rW   c                      e Zd Zy)%test_tokenize_dataclass.<locals>.SubANr   rC   r@   r    SubAr?    r   r@   r@  r  rQ   )
r  r4   r  r$  is_dataclassmake_dataclassr   r  rZ   r"  )a1a2b1r@  ADataClassRedefinedDifferentlynvs         r    test_tokenize_dataclassrH    s   	AB	AB2"!3333 
AB*%
)CCCC"!3333z  ##D)))*%)====$q'"nR&8888 &1%?%?U38_-.&" "0Nq0Q!RRRR 
	B2r@   other))rV   r   rW   )r      rW   )r   r   rV   c                X    t        t        ddd            t        t        |        k7  sJ y )Nr   r   rW   )r4   r  )rI  s    r    test_tokenize_rangerL    s(     %2q/*nUE].KKKKr@   c                     t         j                  d      } t         j                  d      }d|d<   t        | g      t        |g      k7  sJ y )Ni  r   rg   )r_   r   r4   rZ  s     r    test_tokenize_numpy_arrayrN    sA    
		$A
		$AAdG1#.!"5555r@   c                 j    t         j                  ddt         j                  gd      } t        |        y )Nr   u   JoséOrn   )r_   rw   nanr4   r[   s    r    $test_tokenize_object_array_with_nansrR    s'    
%BFF+37A1r@   r   rQ      arz   y              ?rC   c                    t        |        y r:   r8  r   s    r    test_tokenize_base_typesrU    s     1r@   c                 b    t        t        ddg            t        t        ddg            k7  sJ y )Nr   rV   rW   )r4   r   rC   r@   r    test_tokenize_literalrW    s,    '3(+,wQx?P0QQQQr@   z+ignore:the matrix:PendingDeprecationWarningc                    t         j                  j                  d      } t         j                  | j	                  d            }|j                         }t        |      t        |      k(  sJ d|d d t        |      t        |      k7  sJ y )Nrf   r  rV   r   )r_   r;   rh   asmatrixrandcopyr4   )rngrQ   rR   s      r    test_tokenize_numpy_matrixr]    st     ))


%C
CHHSM"A	A!q 1111AcrF!q 1111r@   znot spcls_name)dokc                Z   t         j                  j                  d      }t        j	                  dd|      j                  |       }|j                         }t        |      t        |      k(  sJ t        |d      rd|j                  d d n| dk(  rd|d<   nt        t        |       t        |      t        |      k7  sJ |j                         j                  d	      }t         j                  d      |j                  d d |j                  |       }t        |      t        |      k7  sJ y )
Nrf   r   r  )random_statedatarV   r_  )rX   rX   coo)r_   r;   rh   sprZ  asformatr[  r4   hasattrrb  
ValueErrorr   row)r^  r\  rQ   rR   s       r     test_tokenize_dense_sparse_arrayri    s    ))


%C
Cc*33H=A	A!q 1111 q&s	U	$1!q 1111 	
% A2AEE#2J	

8A!q 1111r@   c                 x   ddg} | | d<   ddg}||d<   t        |       t        |      k7  sJ g g g}|d   j                  |d          |d   j                  |d          g g g}|d   j                  |d          |d   j                  |d          t        |      t        |      k7  sJ i }||d<   t        |       y )NrV   rW   r   rX   )r4   r+  )rQ   rR   r   r   es        r     test_tokenize_circular_recursionrl  :  s    	
AAAaD 
AAAaD!q 1111 
RAaDKK!aDKK!	RAaDKK!aDKK!!q 1111 	AAaD1r@   ))i  r   r   )r   rl   r   )r   r      c                    t        j                  ddd      }t        j                  |  }t        |      t        |      k7  sJ y )Nr   r   r   )r   r   r4   )rI  rQ   rR   s      r    test_tokenize_datetime_datero  T  s<     	dAr"AuA!q 1111r@   c            
        t        t        j                  ddddt        j                  j                               t        t        j                  dddd             t        t        j                  ddd             t        t        j                  dd             t        t        j                  ddddt        j                  j                              t        t        j                  ddddt        j                  j                              k7  sJ t        t        j                  ddddt        j                  j                              t        t        j                  ddddt        j                  j                              k7  sJ t        t        j                  ddddt        j                  j                              t        t        j                  ddddt        j                  j                              k7  sJ t        t        j                  ddddt        j                  j                              t        t        j                  ddddt        j                  j                              k7  sJ t        t        j                  ddddt        j                  j                              t        t        j                  dddd            k7  sJ y )NrV   rW   rX   r   r   )r4   r   timetimezoneutcrC   r@   r    test_tokenize_datetime_timert  b  s8   8==Aq!X->->-B-BCD8==Aq!,-8==Aq)*8==A&'aAq("3"3"7"78	aAq(2C2C2G2GH	IJ J J aAq("3"3"7"78	aAq(2C2C2G2GH	IJ J J aAq("3"3"7"78	aAq(2C2C2G2GH	IJ J J aAq("3"3"7"78	aAq(2C2C2G2GH	IJ J J aAq("3"3"7"78	aAq1	23 3 3r@   c                 h   g d} ddddt         j                  j                  g}t        t	        |      dz         D ]%  }| |d | z   }t        t        j                   |        ' t        t        j                   dddddddt         j                  j                              t        t        j                   dddddddt         j                  j                              k7  sJ t        t        j                   dddddddt         j                  j                              t        t        j                   dddddddt         j                  j                              k7  sJ t        t        j                   dddddddt         j                  j                              t        t        j                   dddddddt         j                  j                              k7  sJ t        t        j                   dddddddt         j                  j                              t        t        j                   dddddddt         j                  j                              k7  sJ t        t        j                   dddddddt         j                  j                              t        t        j                   dddddddt         j                  j                              k7  sJ t        t        j                   dddddddt         j                  j                              t        t        j                   dddddddt         j                  j                              k7  sJ t        t        j                   dddddddt         j                  j                              t        t        j                   dddddddt         j                  j                              k7  sJ t        t        j                   dddddddt         j                  j                              t        t        j                   dddddddd             k7  sJ y )	NrU   r   r   r   rl   rV   rW   rX   )r   rr  rs  r  r   r4   )requiredoptionalr   r*   s       r    test_tokenize_datetime_datetimerx  ~  s   H1aH--112H3x=1$% 1(2A,&x(($/01
 !Q1aAx/@/@/D/DE	))!Q1aAx?P?P?T?TU	VW W W !Q1aAx/@/@/D/DE	))!Q1aAx?P?P?T?TU	VW W W !Q1aAx/@/@/D/DE	))!Q1aAx?P?P?T?TU	VW W W !Q1aAx/@/@/D/DE	))!Q1aAx?P?P?T?TU	VW W W !Q1aAx/@/@/D/DE	))!Q1aAx?P?P?T?TU	VW W W !Q1aAx/@/@/D/DE	))!Q1aAx?P?P?T?TU	VW W W !Q1aAx/@/@/D/DE	))!Q1aAx?P?P?T?TU	VW W W !Q1aAx/@/@/D/DE	))!Q1aAtD	EF F Fr@   c                     d} t        j                  t        j                  dt	        j
                  |       g      }|j                          y )Na  
    def inc(x):
        return x + 1

    inc2 = inc
    def sum(x, y):
        return x + y

    from dask.base import tokenize
    assert tokenize(inc) != tokenize(sum)
    # That this is an alias shouldn't matter
    assert tokenize(inc) == tokenize(inc2)

    def inc(x):
        return x + 1

    assert tokenize(inc2) != tokenize(inc)

    def inc(y):
        return y + 1

    assert tokenize(inc2) != tokenize(inc)

    def inc(x):
        y = x
        return y + 1

    assert tokenize(inc2) != tokenize(inc)
    z-c)
subprocessrunsys
executabletextwrapdedentcheck_returncode)scriptprocs     r    test_tokenize_functions_mainr    s9    F: >>3>>41HIJDr@   c                     t        j                  ddt        t        j                  d      fgdd i      }  | d       | d	      }}t	        |      t	        |      k7  sJ y )
NrD   paramF)reprrG   c                    | j                   S r:   )r  r=   s    r    r   z7test_tokenize_dataclass_field_no_repr.<locals>.<lambda>  s
    TZZ r@   )	namespacerV   rW   )r$  rB  r   r%  r4   )rD   rC  rD  s      r    %test_tokenize_dataclass_field_no_reprr    sa    ""
5+++7	89&(?@	A qT1Q4B"!3333r@   c                 n    t        t        j                        t        t        j                        k7  sJ y)ziTop-level functions in the operator module have a __self__ attribute, which is
    the module itself
    N)r4   operatoraddmulrC   r@   r    test_tokenize_operatorr    s$     (,,'>(,,+GGGGr@   c                 8   t        j                  d      } t        j                  d      }t        j                  d      }t        |       t        |      k(  sJ t        |       t        |      k7  sJ | j                          t        |       t        |      k7  sJ y Nr6   i  r;   Randomr4   rO  s      r    test_tokenize_random_stater    s}    cAcAcA!q 1111!q 1111HHJ!q 1111r@   c                 t   t         j                  j                  d      } t         j                  j                  d      }t         j                  j                  d      }t        |       t        |      k(  sJ t        |       t        |      k7  sJ | j                          t        |       t        |      k7  sJ y r  r_   r;   rh   r4   rO  s      r     test_tokenize_random_state_numpyr    s    
		c"A
		c"A
		c"A!q 1111!q 1111HHJ!q 1111r@   moduler;   z	np.random)marksc                L   t        |       } | j                  }t        j                  t        j                  |            }t        |      t        |      k(  sJ  |        t        j                  t        j                  |            }t        |      t        |      k(  sJ t        |      t        |      k7  sJ | j                  d       t        j                  t        j                  |            }t        |      t        |      k(  sJ t        |      t        |      k7  sJ y)zrandom.random() and other methods of the global random state do not compare as
    equal to themselves after a pickle roundtripr6   N)evalr;   pickler(   r)   r4   seed)r  rQ   rR   r   r   s        r    test_tokenize_random_functionsr    s     &\FAV\\!_%A!q 1111 CV\\!_%A!q 1111!q 1111
KKV\\!_%A!q 1111!q 1111r@   c                     t        j                  d      j                   } t        j                  d      j                   }t        |       t        |      k7  sJ y r  r  ra  s     r    )test_tokenize_random_functions_with_stater    sB    c!!Ac!!A!q 1111r@   c                     t         j                  j                  d      j                  } t         j                  j                  d      j                  }t        |       t        |      k7  sJ y r  r  ra  s     r    /test_tokenize_random_functions_with_state_numpyr    sN    
		c"))A
		c"))A!q 1111r@   znot pac                     t         j                         } t         j                         }t        |       t        |      k7  sJ y r:   )par`   rc   r4   ra  s     r    &test_tokenize_pyarrow_datatypes_simpler    s0    

A


A!q 1111r@   c                 &   t         j                  t         j                         t         j                         d      } t         j                  t         j	                         t         j                         d      }t        |       t        |      k7  sJ y )NrZ  )r  structra   rv  rc   int16r4   ra  s     r    'test_tokenize_pyarrow_datatypes_complexr    sX    
		
56A
		

288:67A!q 1111r@   c                 j     G d d      } t         | ddg            t         | ddg            k7  sJ y )Nc                      e Zd Zd Zy)3test_tokenize_opaque_object_with_buffers.<locals>.Cc                8    t         j                  |      | _        y r:   )r_   rw   r   r  s     r    r?   z<test_tokenize_opaque_object_with_buffers.<locals>.C.__init__*  s    XXa[DFr@   Nr  rC   r@   r    r   r  )  s    	!r@   r   rV   rW   rX   r8  r  s    r    (test_tokenize_opaque_object_with_buffersr  &  s6    ! ! !QF)$q!Qy(AAAAr@   c                  $    e Zd Zd Zd ZexZxZZy)
NumbaDummyc                     y)NFrC   r=   s    r    __bool__zNumbaDummy.__bool__3  r7  r@   c                    d }|S )Nc                    | S r:   rC   r9  s    r    wrapperz,NumbaDummy._dummy_decorator.<locals>.wrapper7  s    r@   rC   )r>   r*   r+   r  s       r    _dummy_decoratorzNumbaDummy._dummy_decorator6  s     Nr@   N)rH   rI   rJ   r  r  jit	vectorizeguvectorizerC   r@   r    r  r  2  s    		 )988i+r@   r  nopythonc                    | |z   S r:   rC   rZ  s     r    	numba_jitr  A      q5Lr@   
f8(f8, f8)c                    | |z   S r:   rC   rZ  s     r    numba_jit_with_signaturer  F  r  r@   c                    | |z   S r:   rC   rZ  s     r    numba_vectorizer  K  r  r@   c                    | |z   S r:   rC   rZ  s     r    numba_vectorize_with_signaturer  P  r  r@   f8,f8,f8[:]	(),()->()c                    | |z   |d<   y Nr   rC   r   r   outs      r    numba_guvectorizer  U  s    UCFr@   z	not numbac                6     | dd      dk(  sJ t        |        y )NrV   rW   rX   r8  r9  s    r    test_tokenize_numbar  c  s     1:??4r@   c                     t         D  cg c]  } t        |        }} t        |      t        t        |            k(  sJ y c c} w r:   )all_numba_funcsr4   r   r&   )r  r@  s     r     test_tokenize_numba_unique_tokenr  j  s;    />?tnT"?F?v;#c&k**** @s   A c                    t         j                  d      d        } t         j                  dd      d        }t         j                  d      d        }t         j                  dd      d        }t         j                  dgd	      d
        }| ||||g}|D cg c]  }t	        |       }}t        |      t        t        |            k(  sJ y c c}w )NTr  c                    | |z   S r:   rC   rZ  s     r    	local_jitz#test_numba_local.<locals>.local_jitr      1ur@   r  c                    | |z   S r:   rC   rZ  s     r    local_jit_with_signaturez2test_numba_local.<locals>.local_jit_with_signaturev  r  r@   c                    | |z   S r:   rC   rZ  s     r    local_vectorizez)test_numba_local.<locals>.local_vectorizez  r  r@   c                    | |z   S r:   rC   rZ  s     r    local_vectorize_with_signaturez8test_numba_local.<locals>.local_vectorize_with_signature~  r  r@   r  r  c                    | |z   |d<   y r  rC   r  s      r    local_guvectorizez+test_numba_local.<locals>.local_guvectorize  s    QAr@   )r   r  r  r  r4   r   r&   )r  r  r  r  r  r,  r  r@  s           r    test_numba_localr  p  s    
YYY  YY|dY+ , __d_# $ __\D_1 2 4 5 	 &I 099tnT"9F9v;#c&k**** :s   Cr  r|   )
__future__r   r$  r   r   r  r   r  r;   rz  r|  r~  enumr   r   r   r   typingr   r'   r   tlzr	   r
   r   r$   r   r   r   r   	dask.corer   
dask.utilsr   dask.utils_testr   r   ddr_   rd  r  r^  r   fixturer!   r4   rS   r\   markskipifrd   rj   rq   rt   rx   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r-  r/  getattrrZ   r   __str__r   r   r?  parametrizer:  r=  rA  xfailrH  rM  r   r   r   rT  rV  rb  rg  rk  ro  r~  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  	dataclassr  r  r"  r'  r<  rH  rL  rN  rR  r  rU  rW  filterwarningsr]  ri  rl  ro  rt  rx  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  rC   r@   r    <module>r     s	   "         
  - -    ' '  B B   *L!$%GN#IHw  ":>(
 HH H HI I H; ; H- - HF F HQ Q HJ J H " H7 7 H< <2 HE E8 H	6 	6 H7 7" H1 1./62
'/>  HV  HH	JJ	  !1!34 5 !34 5+ OPeT]36 4 Q66			4L H2 2  H  H  H  H 0 H  H
 
 H M<<+*%02 >$ &	( 		L&4,P2'	4
4 tWgt&DEC FC       + + + 
S.: LL H6 6 H 
 !T3c2tRRsC	HR HIJ2 K 2 HX.2 / 224 2238'FTD	4H2 H2 2 |v||Kv{{/A/A(/KLM2	2*2 H2 2 H2 2 H2 2 HB B 
9 
9 LE D  <$' ( $   - . M?K0 1
 " K 1 2 !
 K + !+
 K + !+r@   