
    tfG                    x   d dl mZ 	 d dlZd dlmZmZmZ d dlmZm	Z	m
Z
 d dlmZmZmZmZmZ d dlmZmZmZmZmZ d dlmZ  G d d	e      Ze	 d	 	 	 	 	 	 	 dd
       Ze	 ddd	 	 	 	 	 	 	 dd       Z	 ddd	 	 	 	 	 	 	 ddZ	 	 	 	 	 	 ddZ	 	 	 	 	 	 ddZ edd      Z	 	 d	 	 	 	 	 	 	 ddZddZd dZ d!dZ!y)"    )annotationsN)defaultdictdeque
namedtuple)IterableMappingMutableMapping)AnyCallableLiteral
NamedTupleoverload)get_dependenciesget_depsgetcycleistaskreverse_dict)Keyc                  "    e Zd ZU ded<   ded<   y)Orderintpriorityzfloat | intcritical_pathN)__name__
__module____qualname____annotations__     T/var/www/html/software/conda/envs/higlass/lib/python3.12/site-packages/dask/order.pyr   r   :   s    Mr   r   c                    y Nr   dskdependenciesreturn_statss      r    orderr'   ?        r   F)r&   c                    y r"   r   r#   s      r    r'   r'   I   r(   r   c               
  &'()*+,-./0123456789:;<=>?@ | si S t        |       } t        |       }d}| D ci c]  }|t        | |       c}nd}t              t              --j	                         D ch c]
  \  }}|r	| c}}1j	                         D ch c]
  \  }}|r	| c}}8i 7d}d}t        t              6|sid}t        1      D ]  }	|	8v rt        | |	         rt        |	         dkD  s)d}t        |       dz
  |z
  }
rt        |
d      7|	<   n|
7|	<   |dz  }1j                  |	       | |	= -|	= |	   D ]-  }-|   j                  |	       -|   r1j                  |       /  t        8      D ]  }|1v rt        | |         rt        -|         dkD  s)|sd}t        j                        8j                  |       -|   D ]A  }6|   j                  |       |   j                  |       |   r18j                  |       C | |= |= -|=  |sit        -      \  4@t        @      t        |       k7  r0t        | d      }t!        ddj#                  d	 |D              z        J t%        -      \  93t%        -      \  2}d.t               ;t               5g :i //j&                  0d,t               }d&i 'd'39@fd
?d&(,-.14567:;fd(d&fd}|d(-/0147:;?fd       }d128fd} |       }dd}|sd124579;?fd} |       }nt)        1?d      }|j&                  }~g }|j*                  )|j                  <d)<fd}|j,                  >|j.                  *d*>fd}|j&                  +|j0                  =d+=fd}t        7      |k  ri,dz  ,|rJ |rJ  |       }|   } ||       |r%t3        |?      } ||       |   } ||       |r%d}|r |       }|7v r4|   r|/v r 0|      D ]  } |t5        |              7 ||       |   j7                  7      } g }!g }"|"j*                  }#|!j*                  }$t)        | ?      D ]  }%|%/v r	 |#|%        |$|%        t        |!      dkD  rd}|!D ]
  }% ||%        |"D ]   }% 0|%      D ]  } |t5        |              " ~ |rt        :      t        |      k  r |         (|       |r |        t        7      |k  rit        7      |k(  sJ 7S c c}w c c}}w c c}}w )a   Order nodes in dask graph

    This produces an ordering over our tasks that we use to break ties when
    executing.  We do this ahead of time to reduce a bit of stress on the
    scheduler and also to assist in static analysis.

    This currently traverses the graph as a single-threaded scheduler would
    traverse it.

    Examples
    --------
    >>> inc = lambda x: x + 1
    >>> add = lambda x, y: x + y
    >>> dsk = {'a': 1, 'b': 2, 'c': (inc, 'a'), 'd': (add, 'b', 'c')}
    >>> order(dsk)
    {'a': 0, 'c': 1, 'b': 2, 'd': 3}
    NTFr      z2Cycle detected between the following keys:
  -> %sz
  -> c              3  2   K   | ]  }t        |        y wr"   )str).0xs     r    	<genexpr>zorder.<locals>.<genexpr>   s     3SV3s   c                    	 |    S # t         $ r= J |    t        |          t        |          |     t        |       fx| <   }|cY S w xY wr"   )KeyErrorlenr.   )r0   rv_sort_keys_cacher%   max_dependentsroots_connectedtotal_dependenciess     r    sort_keyzorder.<locals>.sort_key   s~    	#A&& 	+++"1%LO$OA&'"" A	( 	Q" I	s    AAAc                   | g}|r|j                         } j                  |        
j                  |        j                  |        | v rJ|    r |    j                                |    rrt        z
        | <   n| <   dz  j                  | d      D ]Z  }	|xx   dz  cc<   
j	                  |       	|   r't        |          dk(  r|j                  |       Jj                  |       \ |ry y )Nr+   r   )popdiscardr   getaddr4   append)item
next_itemsdep_crit_path_counter_offsetadd_to_resultcrit_path_counter
dependentsi
leaf_nodes
num_neededreachable_hullrequires_data_taskresultr&   runnablerunnable_hulls      r    rE   zorder.<locals>.add_to_result   s    V
>>#D!!$'""4(t$v~$T*06::<= %T* $Q(9<U(UVt tFA "~~dB/ -31$""3'!#:d+,1"))#. ,-+ r   c                     d fd}|S )Nc                 .    d	  | i | dy # dw xY w)Ng      ?r   r   )argskwargsrD   funcs     r    wrapperz,order.<locals>._with_offset.<locals>.wrapper  s(    (+%.d%f%,-)A)s    )rR   r
   rS   r
   returnNoner   )rT   rU   rD   s   ` r    _with_offsetzorder.<locals>._with_offset  s    	. r   c                    rNj                         } j                          | r&| j                         }|v s|v r|v r	 |       )|g}t        |g      }|r|j	                         }	 |d   }j                  |       |   }|   }|s|   dk  r|D ]
  } |        nt        |      dk(  st        |      dk(  rt        |      dkD  rXt        |      D ]F  }t        |         dk(  s|j                         }	|	j                  |       |j                  |	       H nj                  |       |j                  t        |             |v r|   j                  |       t        |         |   k\  rt               }
 |      D ]  }|d   vs|
j                  |        t        |
      |   k  rt        |
      |<   nl|
rj|
j	                         }|D ]"  }|   r|
j                  |        n
 |       $ |
r:n/t        |         dkD  r|   dk  r|D ]
  } |        n|g|<   	 |r| r&rMyy)aE  Compute all currently runnable paths and either cache or execute them

        This is designed to ensure we are running tasks that are free to execute
        (e.g. the result of a splitter task) not too eagerly. If we executed
        such free tasks too early we'd be walking the graph in a too wide /
        breadth first fashion that is not optimal. If instead we were to only
        execute them once they are needed for a final result, this can cause
        very high memory pressure since valuable reducers are executed too
        late.

        The strategy here is to take all runnable tasks and walk forwards until
        we hit a reducer node (i.e. a node with more than one dependency). We
        will remember/cache the path to this reducer node.
        If this path leads to a leaf or if we find enough runnable paths for a
        reducer to be runnable, we will execute the path.

        If instead of a reducer a splitter is encountered that is runnable, we
        will follow its splitter paths individually and apply the same logic to
        each branch.
        r,   r+   keyN)copyclearr<   r   popleftr?   r4   sortedr@   updateextendlist)
candidatesr[   pathbranchescurrentdeps_downstreamdeps_upstreamkdbranchpruned_branchesrE   r%   rG   known_runnable_pathsknown_runnable_paths_poprI   rJ   rM   rN   rO   r:   s              r    process_runnablesz order.<locals>.process_runnables  s   , !JNN nn&-'3&=*$!#&u $=#++-D"&r(%))'2*4W*=(4W(=.  *'2a7)- !5A$1!$4!5 Y!^s=/AQ/F"?3a7)/X)N !@A (+<?';q'@15(.a(8(0(?!@ !&)00A KKH(MN$$(<<09@@F #$8$A B#-g#6!7 EJG,DW,M !ED'+Bxv'=(7(>(>t(D!E $'#7*W:M#MDH(7E&$8$A +:/>/F/F/H15 )=A/9!}0?0F0Ft0L05,9!,<	)= +: !$L$9 :Q >$.w$71$<)- !5A$1!$4!5 BF 4W =w   r   c                     d} t        t              t              z
        t              z  dk  r+D ]&  }| st        |         } | t        |         k7  s& y y)Nr   g?FT)absr4   )sizerrI   leafs_connected
root_nodess     r    use_longest_pathzorder.<locals>.use_longest_path  sj     s:Z01C
OCcI   !q12DS!344 	! r   c                     t               r"   )NotImplementedErrorr   r   r    
get_targetzorder.<locals>.get_target  s    !##r   c                 \  	
 t        t              } D ]  }|   D ]  }| |xx   dz  cc<     t        t              }| j                         D ]  \  }}||   j	                  |        i 	|j                         D ]  \  }}t        |d      	|<    ~~ d	fd
d
fd}|S )Nr+   Tr[   reversec                 t    r5t              } |    d   }|v r|    j                          |    s| = 5|S y )Nr,   )maxr<   )r[   picked_rootoccurences_grouped_sortedrM   s     r    	pick_seedz3order.<locals>._build_get_target.<locals>.pick_seed  sU    /78C";C"@"DK"f,1#6::<8= 9# > &&r   c                     } 	}r	fd}| z  } nr	fd}| z  } | s        x}r|   } nxs } t        | |      S )Nc                    |     |       fS r"   r   rj   rJ   r:   s    r    <lambda>zForder.<locals>._build_get_target.<locals>.get_target.<locals>.<lambda>      jmXa[%A r   c                    |     |       fS r"   r   r   s    r    r   zForder.<locals>._build_get_target.<locals>.get_target.<locals>.<lambda>  r   r   rZ   )min)
rd   skeyseedrI   ru   rJ   r   rK   rO   r:   s
      r    rz   z4order.<locals>._build_get_target.<locals>.get_target  sb    '
!) AD!.!;J#AD!/*!<J!({*t*%4T%:
%2%Dn
:400r   )rV   z
Key | NonerV   r   )r   r   setitemsr?   r`   )occurrencestrt   occurences_groupedrootoccrj   vrz   r   r   rI   ru   rJ   rK   rM   r8   rO   r:   s            @@r    _build_get_targetz order.<locals>._build_get_target  s    1<S1AK ((+ (ANa'N(( "-S!1(..0 2	c"3'++D12(*%*002 U1/5aXt/T)!,U"K
1 1& r   r|   c                &     |         |        y r"   r   )rA   cpath_append
scpath_adds    r    path_appendzorder.<locals>.path_append  s    T4r   c                &     |         |        y r"   r   )r   cpath_extendscpath_updates    r    path_extendzorder.<locals>.path_extend  s    Uer   c                 &            }  |        | S r"   r   )rA   	cpath_popscpath_discards    r    path_popzorder.<locals>.path_pop#  s    {tr   rZ   )r0   r   rV   ztuple[int, int, int, int, str])rA   r   rV   rW   )rT   Callable[..., None]rV   r   rV   rW   )rV   boolr   )rV   zCallable[[], Key])r   zIterable[Key]rV   rW   )dictr4   r   r   r   r   r   rc   r   r   remover?   r]   deepcopyndependenciesr   RuntimeErrorjoin_connecting_to_rootsr<   r`   r@   ra   rb   r=   r   reversed
difference)Ar$   r%   r&   expected_lendependencies_are_copyrj   r   	all_tasksn_removed_leavesleafpriorC   r   cycle_
scrit_pathrX   rp   rw   longest_pathrz   r   leaf_nodes_sortedr   r   r   r   target	next_depsrA   walked_backre   depsunknownknownk_append	uk_appendrk   rD   r6   rE   r   r   r   rF   rG   rH   rn   ro   rI   ru   r7   rJ   rK   rL   rM   rv   r8   rN   rO   r   r   r   r:   r9   sA    ``                                   @@@@@@@@@@@@@@@@@@@@@@@@@@@r    r'   r'   S   s   . 	
s)Cs8L $=@A+C33A
 !&L)l+J * 0 0 2<1!!<J , 2 2 4>1A!>J%'F I$S)	$ 	,Dz!3t9%T*+a/!	 3x!|&66#(r?F4L#'F4L A% !!$'It$'- ,CsO**40%c?"s+,)	,2 $ 	%Dz!#d)$Z-=)>)B,,0)#'==#>L!!$'%d+ ,C&s+//5 %,,T2',"s+	,
 I &t$	%7 X &3<%L"J"
#c(*d#Ann3U334
 	
 ###&:<&T#O^-j,GOQ	AEMUNH793775J-.BD ""- "- "-H" \ \ \L" $%L$ ,	 ,	\ '(
":8UK&**
N  "M ''LJ %%M ''L !!I''N
 f+
$Q   ~  (	Fyh/D$T*I	"	  :Dv~$// 8 > 4#HTN34D!#D)44V<%'#% <<#NN	(3 %A00 !!	%
 w<!#"&K  #AN# 4A 8 ; 4#HTN344 3x=3}3E#E%'d#E F 	m f+
$p v;,&&&M_ B =>s    U
UU)
U4Uc                .   i }g }| j                         D ci c]  \  }}|s	|t        |       }}}i }t               }| j                         D ]]  \  }}|r	|j                  |       |h||<   ||   }	t        |	      ||<   |	D ]&  }
||
xx   dz  cc<   ||
   r|j	                  |
       ( _ |r|j                         }||   D ]&  }||xx   dz  cc<   ||   r|j	                  |       ( d}d}d}| |   D ]g  }
||
   }|s|}||
   ||<   |r||u r|j                  |      r.d}|s|j                         }t        ||
   ||         ||<   |j                  |       i ||J |xs |||<   |rt               }|D ]  }|||<   	 ||fS c c}}w )a  Determine for every node which root nodes are connected to it (i.e.
    ancestors). If arguments of dependencies and dependents are switched, this
    can also be used to determine which leaf nodes are connected to which node
    (i.e. descendants).

    Also computes a weight that is defined as (cheaper to compute here)

            `max(len(dependents[k]) for k in connected_roots[key])`

    r+   NTF)
r   r4   r   r?   r@   r<   issubsetr]   r   ra   )r%   rG   rM   rg   rj   r   rJ   r7   rootsr   childr[   parentnew_setidentical_setsresult_firstr_child	empty_setrt   s                      r    r   r   d  s    FG(4(:(:(<B1!SV)BJBNEE""$ *1 IIaLF1Ia=D #D	N1 *5!Q&!!%(NN5)**  kkm o 	'Fv!#f%v&	' !#& 	(EUmG&&4U&;s# !W,0@0@0N!&*//1G&).*?PSAT&Us#w'	(  "l&>>>-s; B %I q	>!!o Cs
   
FFc                   i }i | j                         D ]  \  }}t        |      ||<   |rd|<    |j                         }g }|j                  }|j                  }D ]'  }	||	   D ]  }
||
xx   dz  cc<   ||
   r ||
        ) |rL |       }	dt        fd| |	   D              z   |	<   ||	   D ]  }
||
xx   dz  cc<   ||
   r ||
        |rL|fS )aG  Number of total data elements on which this key depends

    For each key we return the number of tasks that must be run for us to run
    this task.

    Examples
    --------
    >>> inc = lambda x: x + 1
    >>> dsk = {'a': 1, 'b': (inc, 'a'), 'c': (inc, 'b')}
    >>> dependencies, dependents = get_deps(dsk)
    >>> num_dependencies, total_dependencies = ndependencies(dependencies, dependents)
    >>> sorted(total_dependencies.items())
    [('a', 1), ('b', 2), ('c', 3)]

    Returns
    -------
    num_dependencies: Dict[key, int]
    total_dependencies: Dict[key, int]
    r+   c              3  (   K   | ]	  }|     y wr"   r   )r/   r   rM   s     r    r1   z ndependencies.<locals>.<genexpr>  s     KfUmKs   )r   r4   r]   r<   r@   sum)r%   rG   rJ   rj   r   num_dependenciesrg   current_popcurrent_appendr[   r   rM   s              @r    r   r     s   , JF""$ 1A
1F1I
 "(G++K^^N ' o 	'Fv!#f%v&	''
 m#Kc9JKKKs o 	'Fv!#f%v&	'  V##r   	OrderInfo)r'   agenum_data_when_runnum_data_when_releasednum_dependencies_freedc                   |t        |       \  }}nt        |      }|J |t        | |d      }g }d}i }i }i }i }	|j                         D 
ci c]  \  }
}|
t	        |       }}
}t        t        | |j                              D ]t  \  }}
|j                  |       |||
<   d}||
   D ]-  }||xx   dz  cc<   ||   dk(  s|||   z
  ||<   |||<   |dz  }/ ||	|
<   ||
   r	||dz
  z  }fd||
<   |||
<   ||z  }v |j                         D 
ci c]!  \  }
}|
t        |||
   ||
   ||
   |	|
         # }}
}||fS c c}}
w c c}}
w )a  Simulate runtime metrics as though running tasks one at a time in order.

    These diagnostics can help reveal behaviors of and issues with ``order``.

    Returns a dict of `namedtuple("OrderInfo")` and a list of the number of outputs held over time.

    OrderInfo fields:
    - order : the order in which the node is run.
    - age : how long the output of a node is held.
    - num_data_when_run : the number of outputs held in memory when a node is run.
    - num_data_when_released : the number of outputs held in memory when the output is released.
    - num_dependencies_freed : the number of dependencies freed by running the node.
    F)r%   r&   r   rZ   r+   )
r   r   r'   r   r4   	enumerater`   __getitem__r@   r   )r$   or%   rG   pressurenum_in_memoryr   runpressurereleasepressurefreedr[   valrJ   rH   releasedrC   r5   s                    r    diagnosticsr     s   $ #+C= j!,/
###y#LuEHM
CKOE0:0@0@0BCHC#s3x-CJCF3AMM:; &3&(C$ 	CsOq O#!#qv:C'4$A	 c
c?X\)MCH#0OC X%M!&, 		
 C 	YS;s+_S-A5:
 	

B 
 x<3 D&
s   E &Ec                      y r"   r   r   r   r    _fr   &  s    r   c           	        t        |       rt        | d         sJ g }| dd  D ]  }t        |t        t        f      r|j                  |       +t        |t              r8t        |      r|j                  t        |             a|j                  |       st        |t              s|j                  |D cg c]  }t        |       c}        t        g|S t        | t              rt        | fS t        g| S c c}w )Nr   r+   )
r   callable
isinstancer.   r   r@   tuple_convert_taskrc   r   )tasknew_speceles       r    r   r   *  s    d|Q    qr( 		@B"sCj)#B&":OOM"$56OOB'B%2 >aq!1 >?		@ X	D%	 Dz{T{ !?s   6C=
c                    i }| j                         D ]  \  }}|}t        |      ||<    t        |      t        |       k7  rt        d      |S )zTake a dask graph and replace callables with a dummy function and remove
    payload data like numpy arrays, dataframes, etc.
    z)Sanitization failed to preserve topology.)r   r   r   r   )r$   newr[   valuesnew_keys        r    sanitize_dskr   ?  sX     Cyy{ -V$V,G- }%FGGJr   r"   )r$   Mapping[Key, Any]r%   Mapping[Key, set[Key]] | Noner&   zLiteral[True]rV   zdict[Key, Order])r$   r   r%   r   r&   zLiteral[False]rV   zdict[Key, int])r$   r   r%   r   r&   r   rV   z!dict[Key, Order] | dict[Key, int])r%   Mapping[Key, set[Key]]rG   r   rV   z*tuple[dict[Key, set[Key]], dict[Key, int]])r%   r   rG   r   rV   z%tuple[dict[Key, int], dict[Key, int]])NN)r$   MutableMapping[Key, Any]r   zMapping[Key, int] | Noner%   z$MutableMapping[Key, set[Key]] | NonerV   z&tuple[dict[Key, OrderInfo], list[int]]r   )r   r
   rV   r
   )r$   r   rV   r   )"
__future__r   r]   collectionsr   r   r   collections.abcr   r   r	   typingr
   r   r   r   r   	dask.corer   r   r   r   r   dask.typingr   r   r'   r   r   r   r   r   r   r   r   r   r    <module>r      s   "-\  6 6 = = ? ? P P J 
 
 37	/  	
  
 
 37 $)		/ !	
  
 37N 	N	N/N 	N
 'NbF"(F"6LF"/F"R.$(.$6L.$*.$b 		 #'9=9	!99 79 ,	9x*r   