>ieddlmZddlZddlZddlmZddlmZddlm Z m Z m Z ddl m Z mZejddg d Zd6d Zd ZdZdZdZdZdZdZdZdZdZdZdZdZdZdZ dZ!dZ"dZ#dZ$d Z%d!Z&d"Z'd#Z(d$Z)d%Z*d&Z+d'Z,d(Z-d)Z.d*Z/d+Z0d,Z1d-Z2d.Z3ej45d/d0d gd1Z6d2Z7d3Z8d4Z9d5Z:dS)7) annotationsN)collections_to_dsk)get_deps) diagnostics ndependenciesorder)addincabcdeedcba)paramsc|jSN)param)requests 5lib/python3.11/site-packages/dask/tests/test_order.pyr r s =Fc,t|||kS)N)reverse)sorted)Lrs rissortedrs !W % % % **rcdSr)argss rfrsDrc |\}}}}fdtdD}||dftdfdff|dftdfdffit|}t |df|dfz dksJt |df|dfz dksJfdtdD}||dftdfdff|dftdfdffit|}t |df|dfz dksJt |df|dfz dksJdS)Nc&i|] }|ftfSrr.0ias r z7test_ordering_keeps_groups_together..!(((!!Q!(((rrc&i|] }|ftfSrrr s rr$z7test_ordering_keeps_groups_together.."r%r)rangeupdaterrabs)r bcdeor#s @r#test_ordering_keeps_groups_togetherr3sMAq!Q((((uQxx(((AHHq!fq1a&1a&)Aq6A1v1v3F GHHH aA q!Qy1aV9$ % % * * * * q!Qy1aV9$ % % * * * *((((uQxx(((AHHq!fq1a&1a&)Aq6A1v1v3F GHHH aA q!Qy1aV9$ % % * * * * q!Qy1aV9$ % % * * * * * *rc n|\}}}}}|dftf|dftf|dft|dff|dft|dff|dft|dffi}t|}||df||dfksJ|dftf|dftf|dft|dff|dft|dff|dft|dffi}t|}||df||dfksJ|dftf|dftf|dft|dff|dft|dff|dft|dffi}t|}||df||dfksJdS)zV b0 b1 b2 | \ / a0 a1 a0 should be run before a1 rr'r(Nrrr r#r.r/r0r1dskr2s rtest_avoid_broker_nodesr8*sMAq!Q A A AQF AQF AQF  C c A aV9q!Qy  A A AQF AQF AQF  C c A aV9q!Qy  A A AQF AQF AQF  C c A aV9q!Qy rc|\}}fddD}tdff|df<|fddDd|<t|}|dfdksJ|dfdksJdS) z a3 /| a2 | /| | a1 | | /| | | a0 | | | | | | | b0 b1 b2 b3 \ \ / / c We really want to run b0 quickly c8i|]}|ft|dz f|ffSr'r)r!r"r#r.s rr$z1test_base_of_reduce_preferred..fs2 > > >qAq6A1q5zAq6* > > >r)r'r(r)rc*i|]}|ftdfSr;r)r!r"r.r/s rr$z1test_base_of_reduce_preferred..hs%888aAAq 888r)rr'r(r)r'r)N)rr,r)r r0r1r7r2r#r.r/s @@@rtest_base_of_reduce_preferredr=Us MAq!Q > > > > >I > > >Cq!f+CAKJJ88888<888999 CF c A aV9>>>> aV9>>>>>>rc4|\}}}}}|dft|dff|dft|dff|dft|df|dff|dft|dff|dft|dff|dft|dff|dft|dff|dfd|dfdi }t|}||df||dfksJdS)z a1 | a2 | a3 d1 / \ / b1 c1 | | b2 c2 | c3 r'r(r)Nr5r6s rtest_avoid_upwards_branchingr?qsMAq!Q AQF AQF AQFQF# AQF AQF AQF AQF A A C c A aV9q!Qy rc|\}}}}}|dft|dff|dft|dff|dft|df|dff|dft|dff|dftf|dft|dff|dft|dff|dftf|dft|dff|dft|dff|dft|dff|dft|dff|dft|dffi }t|}||df||dfksJt||df||dfz dksJdS)a a1 | e2 a2 d2 d3 | | \ / e1 a3 d1 \ / \ / b1 c1 | | b2 c2 | c3 Prefer c1 over b1 because c1 will stay in memory less long while b1 computes r'r(r)N)rrr-r6s r$test_avoid_upwards_branching_complexrAsT"MAq!Q AQF AQF AQFQF# AQF A AQF AQF A AQF AQF AQF AQF AQF  C c A aV9q!Qy q!Qy1aV9$ % % * * * * * *rc |\}}}}}|t|||f|t||f|t|f|d|di}t|}||||ksJ||||ksJdS)a{ It's not clear who should run first, e or d 1. d is nicer because it exposes parallelism 2. e is nicer (hypothetically) because it will be sooner released (though in this case we need d to run first regardless) Regardless of e or d first, we should run b before c. a / | \ . b c | / \ | / e d r'r(Nr5r6s r#test_deep_bases_win_over_dependentsrCsx MAq!Q q!QlA1ay!aVQ1a @C c A Q4!A$;;;; Q4!A$;;;;;;rc |\}}}}}|d|t|f|t|f|d|t|fi}t|}||||ksJ||||ksJdS)zq c | e b | | d a Prefer longer chains first so we should start with c r'Nr5r6s rtest_prefer_deeprEsrMAq!Q aQFA1vq!QA 7C c A Q4!A$;;;; Q4!A$;;;;;;rc|dtdD}t|\}}t||dS)Nc4i|]}d|dzztd|zfS)zx%sr'r r!r"s rr$z#test_stacklimit..s+ C C C5AE?S%!), C C Cri')r+rr)r r7 dependencies dependentss rtest_stacklimitrLsA C CeEll C C CC'}}L*, +++++rc|\}}}}}dtdD}t|}t||d<t|}ddi} | dt |D|| ksJdS)Nc&i|]}d|ft|fS)xrHrIs rr$z*test_break_ties_by_str..s" 1 1 1!C8c1X 1 1 1r yci|]\}}|| Srr)r!r"ks rr$z*test_break_ties_by_str..s888daQ888r)r+rlistrr, enumerate) r r#r.r/r0r1r7x_keysr2expecteds rtest_break_ties_by_strrXsMAq!Q 1 1uRyy 1 1 1C C[[FF||CH c ARyH OO88i&7&7888999 ======rcXttdftdftddfddS)Nr'r(rOrQr)rOrZz)rr r )r s r)test_order_doesnt_fail_on_mixed_type_keysr\s/ aS!HCh3G H HIIIIIrcD tjd}d\}}|j||fdx}}t dD]K}|dddddf|dddddfzd}|d}L||dddfzd}t|}t| fd | D} t fd | Dt| d z ksJ fd | D} t fd | Dt| d z ksJt| | ksJdS)N dask.array)c)r'N)sizechunksr(raxis)r'r(c g|] }| Srr)r!rSr2s r z test_gh_3055..s)))!1)))rc3DK|]}|tdz kVdSr(Nlenr!rOr2s r ztest_gh_3055..1))!q3q66A:~))))))rr)c*g|]}|D] }| Srr)r!kkrSr2s rrfz test_gh_3055..s*999"b9919999rc3DK|]}|tdz kVdSrhrirks rrlztest_gh_3055..rmr) pytest importorskiprandomnormalr+cumsumsumdict__dask_graph__r __dask_keys__rjr) daABorigrO_rQwr7rr2s @r test_gh_3055rs  \ * *B DAqyaVI>>>D1 1XX qqq$z]Qqqq!!!Tz] * 2 2 2 : : HH!H   Qqqq$wZ&))A q!! " "C c A))))q(()))A ))))q))) ) )CFFQJ 6 6 6 69999**,,999A ))))q))) ) )CFFQJ 6 6 6 6 !99>>>>>>rcN|\}}}}}|d|dfd||dfdi}t|dS)Nr'r(r)r)r r#r.r/r0r1r7s rtest_type_comparisions_okr s<MAq!Q a!QQ1Iq )C #JJJJJrc |\}}}}}|tf|t|f|t|f|t|f|t|fi}t|}||||ksJ||||ksJdS)z9 a | d b e \ | / c Nr5r6s r test_favor_longest_critical_pathrsxMAq!Q qdA1vq1a&!aVQA ?C c A Q4!A$;;;; Q4!A$;;;;;;rc|\}}}}}d||||fD\}}}} |tf|tf|tf|tf|t||f|t||f|t|f|t||f| t|fi } t| } tt| ddksJ| || |kr| | | |ks&| || |kr| || |ksJdSdS)zX aa / | b d bb dd / \ /| | / a c e cc c3 K|] }|dzV dSrhr)r!rOs rrlz,test_run_smaller_sections../s&22a!e222222rr'r&N)rrmaxr) r r#r.r/r0r1aabbccddr7r2s rtest_run_smaller_sectionsr%sMAq!Q22aAq\222NBB A4 A4 A4 QD Aq!9 Aq!9 QG Q2J QG C c A {3" # #q ( ( ( ( bEAaDLLQrUQqT\\qtad{{qtae||||\\||rc|\}}fddD\}}}fddD\}}}fddD\} } } ||||||| | | g } gfd} || |f|| ||f|| ||f|| |f|| |||f|| ||f| | | f| | | | |f| | | | fi }t|tj|||| g| ksJdS)z c1 | b1 c2 | /| a1 b2 c3 | /| a2 b3 | a3 Prefer to finish a1 stack before proceeding to b2 c3"K|] }|zV dSrrr s rrlz2test_local_parents_of_reduction..S'''A!a%''''''r123c3"K|] }|zV dSrrr!r"r.s rrlz2test_local_parents_of_reduction..Trrc3"K|] }|zV dSrr)r!r"r/s rrlz2test_local_parents_of_reduction..Urrcfd}|S)Nc2dSr)append)rlogrOs rr~z5test_local_parents_of_reduction..f.._\s JJqMMMMMrr)rOr~rs` rrz*test_local_parents_of_reduction..f[s)      rN)rdaskget)r r0r1a1a2a3b1b2b3c1c2c3rWrr7r#r.r/rs @@@@rtest_local_parents_of_reductionrCsMAq!Q'''''''JBB'''''''JBB'''''''JBBBBBB3H C QQrUUH QQrUUBK QQrUUBK QQrUUH QQrUUBO QQrUUBK QQrUUH QQrUUBO QQrUUBK C #JJJHS2r2, (??????rcf|\}}}fddD\ }}}}}}} } } fddD\} } }}| tf| tf|tf|tf|t| f|t| f|t| | f|t| f|t| |f|t|f| t||f| t|f| t|fi }t|dtfddDcxkrdksnJdtfd dDcxkrd ksnJt| | ||gd ksJd S) z a1 a2 a3 a4 a5 a6 a7 a8 a9 \ | / \ | / \ | / \ | / b1 b2 b3 b4 Want to finish off a local group before moving on. This is difficult because all groups are connected. c3"K|] }|zV dSrrr s rrlz(test_nearest_neighbor..~s')E)EA!a%)E)E)E)E)E)Er 123456789c3"K|] }|zV dSrrrs rrlz(test_nearest_neighbor..s',,a!e,,,,,,r1234r)c3VK|]#}|ztdz kV$dSrhri)r!r"r#r2s rrlz(test_nearest_neighbor..s:>>Q1QU8c!ffqj(>>>>>>rr'c3VK|]#}|ztdz kV$dSrhri)r!r"r.r2s rrlz(test_nearest_neighbor..s:99Q1QU8c!ffqj(999999rr&rN)rrrvmin)r r/r~rrra4a5a6a7a8a9rrrb4r7r#r.r2s @@@rtest_nearest_neighborrssMAq!Q)E)E)E)E)E)E)E&BBBBB,,,,V,,,NBB QD QD QD QD QG QG QBK QG QBK QG QBK QG QG C c A s>>>>>+>>>>> B B B B B B B B B B s99999&99999 = = = =A = = = = = = S"b"b! " " #q ( ( ( ( ( (rcltftftfd}t|}|ddddksJdS)z#Prefer ordering tasks by name first)r#r'r#r(r#r)rr'r(Nr5r7r2s rtest_string_orderingrsCTaTaT : :C c A 1Q77 7 7 7 7 7 7rctdftdftdftfd}t|}|dddddksJdS) z=Prefer ordering tasks by name first even when in dependenciesr.)rrrr.rr'r(r))r.rrrNr5rs rtest_string_ordering_dependentsrsOX!SaXQD Q QC c A a1Q?? ? ? ? ? ? ?rc<|\}}}}}|dfd|dfd|dfd|dft|df|df|dff|dfd|dfd|dft|df|df|dffi}t|}||df||dfksJ||df||dfksJ||df||dfksJdS)Nrr'r(r5)r r#r.r/r~r7r2s rtest_prefer_short_narrowrsMAq!Q A A A AQFQFQF+ A A AQFQFQF+ C c A aV9q!Qy aV9q!Qy aV9q!Qy rc|\}}}}}||z}|d|dft|ddf|dft|ddf|dfd|dft|df|df|dff|dft|ddf|dft|ddf|dft|df|df|dffi}t|}||df||dfksJ||df||dfksJ||df||dfksJ||df||dfksJ||df||dfksJdS)a From https://github.com/dask/dask-ml/issues/206#issuecomment-395869929 Two cases, one where chunks of an array are independent, and one where the chunks of an array have a shared source. We handled the independent one "well" earlier. Good: c2 / \ \ / \ \ c1 \ \ / | \ \ \ c0 a0 b0 a1 b1 Bad: c2 / \ \ / \ \ c1 \ \ / | \ \ \ c0 a0 b0 a1 b1 \ \ / / \ \ / / a-b The difference is that all the `a` and `b` tasks now have a common ancestor. We would like to choose c1 *before* a1, and b1 because * we can release a0 and b0 once c1 is done * we don't need a1 and b1 to compute c1. rr'r(Nr5)r r#r.r/r~abr7r2s rtest_prefer_short_ancestorrslLMAq!Q QB A AB1 AB1 A AQFQFQF+ AB1 AB1 AQFQFQF+ C c A aV9q!Qy aV9q!Qy aV9q!Qy aV9q!Qy aV9q!Qy rcj|\}}}}}i|dftf|dft|dff|dft|dff|dft|df|dff|dftf|dft|df|df|dff|dftf|dft|dff|dft|dff|dft|df|df|dff|dftf|dft|df|df|dff|dftf|dft|dff|dft|dff|dft|df|dff}t|}||df||dfks||df||dfksJdSdS)z b1 b3 b5 |\ / | \ / | c1 c2 c3 c4 c5 |/ | \ | / | \| d1 d2 d3 d4 d5 | | | e1 e2 e5 Want to finish b1 before we start on e5 r'r(r)r&Nr5r6s rtest_map_overlaprsMAq!Q  A  AQF   AQF   AQFQF#   A   AQFQFQF+   A  AQF   AQF   AQFQFQF+  A  AQFQFQF+  A  AQF   AQF   AQFQF#! C& c A aV9q!Qy Aq!fI1a& $9$9$9$9 $9$9rc |\}}}idftfdftfdftfdftfdftfdftfdftfdftfd ftfd ftfdftdffdftdfdffd ftdfdfdffdftdfdfdffdftd fdfdffdftdfd fdffdftdfdfd ffd ftdfdfdffdftd fdfdffdftdfd fdffi}t|}tfd |D}tfd |D}|dd vsJ|ddkr|gdksJ|gdksJdS|gdksJ|gdksJdS)zSee https://github.com/dask/dask/issues/5584#issuecomment-554963958 We were using key names to infer structure, which could result in funny behavior. rr'r(r)r&rr c34K|]\\}}}|k|VdSrr)r!letterr~valr#s rrlz.test_use_structure_not_keys..60EE( S1EErc34K|]\\}}}|k|VdSrr)r!rr~rr.s rrlz.test_use_structure_not_keys..7rr>r'r)) rr'r(r&rrrP ) r)rrr ) rr(r&rrrPrrrr) r'r)rrrrrrrrNrrritems)r r~r7r2AsBsr#r.s @@rtest_use_structure_not_keysrs  MAq!Q  A  A  A  A   A   A   A  A  A  A  AQF   AQFQF#  AQFQFQF+  AQFQFQF+  AQFQFQF+  AQFQFQF+! " AQFQFQF+# $ AQFQFQF+ AQFQFQF+ AQFQFQF+)  C, c A EEEEAGGIIEEE E EB EEEEAGGIIEEE E EB a5F???? !uzz77777779999999998888888888888888rc  |\}}} }d}|dftdf|dftdf|dftdf dft|df|df|dffi}td|D];}t|dff|||f<t|dff|||f<t |dz f||f||ff| |f<.Rs0GG*-636Q;;A;;;;GGrN)rr+rrr) r r#r.r/r1depthr7r"r2rWactualr0s @r&test_dont_run_all_dependents_too_earlyrAs%MAq!Q E AA AA AA AQFQFQF+  C 1e__661a&kQF 1a&kQF 1a!e*q!fq!f5QF  c A444H GGGGaggiiGGG G GF v      rc  |\}}} z z i|dfd|dft|dff|dft|dff|dft|dff|dft|dff|dft|dffdft|dffdftdff dft|dff dft dff dftdf dff dft|dff dft dff dft|dff dft dff dft df dff|dft|df|df|df df dff}t|}gd}t fd|D}||ksJ||df||dfdz ksJdS)a=From https://github.com/dask/dask/pull/5646#issuecomment-562700533 Sometimes we need larger or wider DAGs to test behavior. This test ensures we choose the branch with more work twice in successtion. This is important, because ``order`` may search along dependencies and then along dependents. rr'r(r)) r'r(r)r&rrrrrrPc38K|]\\}}}|hv|VdSrr)r!rr~rr0rr1ees rrlz7test_many_branches_use_ndependencies..xs=RR.+61qAr1b>9Q9QA9Q9Q9Q9QRRrNr) r r#r.r/r7r2rWrr0rr1rs @@@@r$test_many_branches_use_ndependenciesrVsjMAq!Q QB QB  A  AQF   AQF   AQF   AQF   AQF   AQF   AQF   Q!aV  Q!b!W  Q!aVb!W%  AQF   AQF   Q!aV  Q!b!W  Q!aVb!W%! " AQFQFQFRGb!W=# C& c A...H RRRRRRRAGGIIRRR R RF X     aV91a& A % % % % % %rctjtd5tjdt dfiddddn #1swxYwYtjtd5t dt dfidddn #1swxYwYtjtd5t dt dfidddn #1swxYwYtjtd5t t dft dft dfddddn #1swxYwYtjtd5t t dft dft ddfd d dddn #1swxYwYtjtd5t t dft dft ddft dfd ddddS#1swxYwYdS) NzCycle detected)matchr#r#rr.r/)r#r.r/r0r')r#r.r/r0)rqraises RuntimeErrorrrrrrrrtest_order_cycler}sw |+; < < <'' #3x#&&&''''''''''''''' |+; < < < sQHo |+; < < <)) x!X'((())))))))))))))) |+; < < <== QHAs81c(;;<<<=============== |+; < < <JJ QHAs81c3-aHHIIIJJJJJJJJJJJJJJJ |+; < < <QQ QHAs81c3-q#hOOPPPQQQQQQQQQQQQQQQQQQskAA A -BBB8CC!$C!)D88D<?D<+FFF;2G::G>G>c0tiiksJdSrrrrrtest_order_emptyrs 99??????rc|\}}}}}i|dfd|dft|dff|dft|dff|dft|dff|dft|df|dff|dft|dff|dft|dff|dft|dff|d ft|dff|dfd|dft|dff|dft|dff|dft|dff|dft|dff|dft|dff|dft|dff|dft|dff}t|}||dfdksJ||df||dfksJ||df||dfksJ||df||dfksJd S) ax a7 a8 <-- do these last | / a6 e6 | / a5 c5 d5 e5 | | / / a4 c4 d4 e4 | \ | / / a3 b3---/ | a2 | a1 | a0 <-- start here Test that we are able to switch to better dependents. In this graph, we expect to start at a0. To compute a4, we need to compute b3. After computing b3, three "better" paths become available. Confirm that we take the better paths before continuing down `a` path. This test is pretty specific to how `order` is implemented and is intended to increase code coverage. rr'r(r)r&rrrrNr5r6s rtest_switching_dependentsrs06MAq!Q  A  AQF   AQF   AQF   AQFQF#   AQF   AQF   AQF   AQF   A  AQF   AQF   AQF   AQF   AQF   AQF ! " AQF # C& c A aV9>>>> aV9q!Qy aV9q!Qy aV9q!Qy rcd}|\}}}}}i}||||g}|D]} || dftdf| dft| dff| ddft| dff| ddft| dffit|D]~\} } || d| ft| ddf| ddff| d| ft| d| ff| d| dft| d| ff| d| dft| d| ff| d| dft| d| dff| d| dft| d| dffit|} d} |D]G} t t |D](} | | d| df| | d| dfz }|dksJ| |z } )H| d ksJt || d}t||ksJt|}|D]5} t t |D]} t| d| dff|| d | df<6t|} d} |D]G} t t |D](} | | d| df| | d | dfz }|dksJ| |z } )H| d ksJt || d}t||ksJt|}|D](} t t |D] } || d| df= )t|} d} |D]G} t t |D](} | | d| df| | d| dfz }|dksJ| |z } )H| d ksJt || d}t||ksJt|}|D](} t t |D] } || d| df= )t|} t || d}t||ksJ|D]M} t t |D].} t| | d| df| | d| dfz dksJ/NdS)zFrom https://github.com/dask/dask/issues/5859#issuecomment-608422198 See the visualization of `(maxima, argmax)` example from the above comment. This DAG has enough structure to exercise more parts of `order` rrr'r(r)r&rr8r2rK-rPN) r,rrUrr+rjrrrwr-)r max_pressurer#r.r/r0r1r7abcrOr"rQr2totalrpressuredsk2dsk3dsk4s r test_order_with_equal_dependentsrs?LMAq!Q C aA,C  AAAQF Aq A1v;Aq A1v;     cNN  DAq JJ1IAq!9q!Qi81IAq!9~1aL1q!Qi.1aL1q!Qi.1aL1q!Ql"31aL1q!Ql"3       c A E s3xx  AQ1aL/Aq!QlO3C7777 SLEE  B;;;;3!$$$Q'H x==L ( ( ( ( 99D 33s3xx 3 3A"#aAq\!2D!Q1   3 d A E s3xx  AQ1aL/Aq!QlO3C7777 SLEE  B;;;;41%%%a(H x==L ( ( ( ( 99D ##s3xx # #AaAq\"" # d A E s3xx  AQ1aL/Aq!QlO3C7777 SLEE  B;;;;41%%%a(H x==L ( ( ( ( ::D ##s3xx # #AaAq\"" # d A41%%%a(H x==L ( ( ( ( @@s3xx @ @Aq!Q1Aq!Q<899R????? @@@rcdddtddftddfddd d d td d ftd dfddtddddfd}t|}|d|d ksJdS)a https://github.com/dask/dask/issues/6745 We have 1. A terminal node that depends on the entire graph ('s') 2. Some shared dependencies near the roots ('a1', 'a4') 3. But the left and right halves are disconnected, other than the terminal node. s / / \ \ / / \ \ s00 s10 s01 s11 | | | | b00 b10 b01 b11 / \ / \ / \ / \ a0 a1 a2 a3 a4 a5 Previously we started at 'a', and worked up to 's00'. We'd like to finish 's00' completely, so we progress to 's' and work through its dependencies. Ideally, we would choose 's10', since we've already computed one of its (eventual) dependencies: 'a1'. However, all of 's00' through 's11' had equal metrics so we fell back to the name tie-breaker and started on 's01' (via 'a3', a4', 'b01', ...). )rr;)r(rrrr.rr.r')r))r&)rrr#r&r#rr.r(r.r)storerrr r'rr rr'r r'r')rrrrrr r rrrrr r rr Nr5rs rtest_terminal_node_backtrackr"s<h)h)!!h)h)!!      #  C2 c A X;8 $ $ $ $ $ $rc tjd tjd}ddl fdtdD} |dd}||}||d }|d ||j d }| |d d }t|j dD}t|fdfd|D fd|D t fdD} t fdD} | | ksJdS)Nr^zarrrcdg|],}dd-S))nr&dr(rb) from_arrayones)r!r"rznps rrfz0test_array_store_final_order..cs5 R R RAbmmBGGH--hm?? R R Rrr&rcrT) overwritedest)namedatarbrF)lockcomputecrg|]4}t|t|dd2|5S)rz store-map-) isinstancetuple startswith)r!rSs rrfz0test_array_store_final_order..ms< U U UAjE22 Uqt|7T7T Ua U U Urc|Srr)rSr2s rz.test_array_store_final_order..ns AaDr)keyc8g|]}|ddk|Srr!rS first_stores rrfz0test_array_store_final_order..os+FFFaQrUk"o-E-E-E-E-Erc8g|]}|ddk|Sr(rr*s rrfz0test_array_store_final_order..ps+III"R0H0H10H0H0Hrc3*K|] \}}|v |VdSrr)r!rSrconnected_storess rrlz/test_array_store_final_order..rs2IIda18H3H3H3H3H3H3HIIrc3*K|] \}}|v |VdSrr)r!rSrdisconnected_storess rrlz/test_array_store_final_order..ss2OOAa;N6N6N16N6N6N6NOOr)rqrrnumpyr+ concatenaterechunkDirectoryStoregroup empty_like chunksizer rrrrr)tmpdirrarraysrOr rootrr0stores connected_maxdisconnected_minr.rzr0r+rr2s @@@@@@rtest_array_store_final_orderr>[s  \ * *B  v & &D R R R R Rq R R RF vA&&..x88A    ' 'E ::et: , ,D ??Qq{d? S SD 5%00A af AV U U U UFf....111KFFFF6FFFIIIIfIIIIIIIaggiiIIIIIMOOOOOOOOO + + + + + + +rcidtddfdtddfdtdd fd td d fd tddfdtddfdtd dfdtddfdtdd fdtdfdtdfdtdfdtdfdtdfdtdfd td!fd"td#fid$td%fd&td'fd(td)fdtd*fd td+fd td,fdtd-fdtd.fd td/fdtd0fdtd1fdtd2fdtd3fd4td(fd5tdfd6td4d5fdtd3fid7tddfd'td,fdtd1fd!td0fd8tddfd9td8d:fd:td&d$fd;td7dd=fdtd.fd#td2fd%td/fdtd;d9fd1td?fd+td@fd-tdAfd*tdBfd/tdCfd2tdDfd3tdEfd.tdFfd,tdGfd0tdHfd?ttfdFttfdDttfdEttfdHttfdBttfttfttfttfttfdI}t|\t|fdJD}fdKfdL|D}t |}t |dMdNdOkst |dPksJdMSdMS)Qahttps://github.com/dask/dask/pull/7929 This graph begins with many motifs like the following: | | c1 c2 \ / b | a We want to compute c2 and c3 pretty close together, because if we choose to compute c1, then we should also compute c2 so we can release b. Being greedy here allows us to release memory sooner and be more globally optimal. a00a06a08a01a28a26a02a24a21a03a22a25a04a29a20a05a23a27a07a09a43a10a36a11a33a12a47a13a44a14a42a15a37a16a48a17a49a18a35a19a46a55a53a60a54a59a56a61a52a57a58a30a31a32a34a38a39a40a41a50a45a51a62a68a70a67a71a64a65a63a69a66)rrrrcNh|]!\}}| |"Sr)pop)r!r&depsrJs r zAtest_eager_to_compute_dependent_to_free_parent..s0WWWic4\RUEVWtxxzzWWWrcJ|\}}t||z Sr)r-)rr#r.r2s rcostz.costs&11Q4!A$;rc4i|]}||Srr)r!r&rrKs rr$zBtest_eager_to_compute_dependent_to_free_parent..s)CCCCS$$z#//CCCrNr))rrrrrvaluesrv)r7parents cost_of_pairscostsrrJrKr2s @@@@r.test_eager_to_compute_dependent_to_free_parentrws6 I  5% I  5% I  5% I  5% I  5% I  5% I  5% I  5% I  5% I  5zI  5zI  5zI  5zI  5zI  5zI  5z!I " 5z#I I $ 5z%I & 5z'I ( 5z)I * 5z+I , 5z-I . 5z/I 0 5z1I 2 5z3I 4 5z5I 6 5z7I 8 5z9I : 5z;I < 5z=I > 5z?I @ 5zAI B 5% CI D 5zEI I I F 5% GI H 5zII J 5zKI L 5zMI N 5% OI P 5% QI R 5% SI T 5% UI V 5zWI X 5zYI Z 5z[I \ 5% ]I ^ 5% _I ` 5zaI b 5zcI d 5zeI f 5% gI I I h 5% iI j 5zkI l 5zmI n 5zoI p 5zqI r 5zsI t 5zuI v 5zwI x 5zyI z 5z{I | 5z}I ~ 1vI @ 1vAI B 1vCI D 1vEI F 1vGI H 1vII I J1v1v1v1vQI I I CT (}}L* c AWWWW:+;+;+=+=WWWG     DCCCC7CCCM ='')) * *E uSbSz??b CJJ"$4$4$4$4 $4$4rc8|\}}}}}|dftf|dftf|dftf|dftf|dftf|dft|df|df|dff|dft|df|df|dff|dft|df|df|dff|dft|df|dff|dft|dffi }t|}||dft|dz ksJ||dft|dz ksJ||dft|dz ksJt|\}} | gdksJd|D|dfd|dfd|dfd|dfd|dfd|dfd|dfd |dfd |dfd |dfd i ksJd |D|dfd|dfd|dfd |dfd|dfd|dfd|dfd|dfd|dfd|dfdi ksJd|D|dfd|dfd|dfd|dfd|dfd|dfd|dfd|dfd|dfd|dfdi ksJd|D|dfd|dfd|dfd|dfd|dfd|dfd|dfd|dfd|dfd|dfdi ksJd|D|dfd|dfd|dfd|dfd|dfd|dfd|dfd|dfd|dfd|dfdi ksJdS)z a1 b1 c1 d1 e1 /|\ /|\ /|\ /| / / | X | X | X | / / |/ \|/ \|/ \|/ a0 b0 c0 d0 e0 rr'r(r)) rr'r(r)r(r)r(r)r(r'c$i|] \}}||jSrrr!r&rs rr$z$test_diagnostics..s 8 8 8xsCC 8 8 8rr&rrrrrc$i|] \}}||jSr)agers rr$z$test_diagnostics..s 6 6 6XS#C 6 6 6rc$i|] \}}||jSr)num_dependencies_freedrs rr$z$test_diagnostics..! I I ISC+ I I Irc$i|] \}}||jSr)num_data_when_runrs rr$z$test_diagnostics..s! D D D83C& D D Drc$i|] \}}||jSr)num_data_when_releasedrs rr$z$test_diagnostics..*rrN)rrrjrr) r r#r.r/r0r1r7r2infomemory_over_times rtest_diagnosticsrsIMAq!Q A A A A A AQFQFQF+ AQFQFQF+ AQFQFQF+ AQFQF# AQF  C c A aV9C1 $ $ $ $ aV9C1 $ $ $ $ aV9C1 $ $ $ $(--D  === = = = = 8 84::<< 8 8 8 A A A A A A A A A A =     7 6 6 6 6 A A A A A A A A A A ;     J IDJJLL I I I A A A A A A A A A A N     E Dtzz|| D D D A A A A A A A A A A I     J IDJJLL I I I A A A A A A A A A A N      rcXtd\}}}}}i}tdD]}|d|ftf|d|ftf|d|ft|d|ff|d|ft|d|f|d|ff|d|ft|d|ff|d|ft|d|ff|d|ft|d|ff|d|ft|d|ffi}||tdD]-}|||ft||df||df||dffi.t |}t ||\} } t | dksJdS)Nr r)rr'r(rr)rTr+rr,rrr) r#r.r/r0r1r7ixpartr2r~rs rtest_xarray_like_reductionr8sMMMAq!Q CAhh  2J 2J 2JQ2J 2JQ2JAr 3 2JQ2J 2JQ2J 2JQ2J 2JQ2J   4Ahh   B!aQZ!RaQZ@     c AcQ'''KAx x==A      roptimizeTc tjd}ddlm}d}|t gd|j|dddfd fgd|j|dddfd f }|d z}|j|jz|d <| d }tt|g|}tt| g|}t|dt|dksJt|ddksJdS)Nxarrayri)timefacejr"r')rPr'r)r)r)anom_uanom_vr(uvr)optimize_graph2)rqrr dask.arrayarrayDatasetrwrsrrmeanrrto_dask_dataframer) rxrrzradsquadr diag_arraydiag_dfs rtest_array_vs_dataframerTsu  X & &B D  ***   $3!5o NN +**   $3!5o NN   B q5DRY&DJ 99V  D/xPPPQQJD22445hOOOG wqz??c*Q-00 0 0 0 0 z!}   " " " " " "rc  tjd}tjd}ddlm}ddlm}|jdd|ddzf }|jdd z}||d d gd d | |jd|zfi}|jdd }||d d gd d | ddzfi}| d  d }| d |z }| d } t| \} } t| dksJdS)Nr1rr) parse_bytes)iir'10MiBrrr&rrOday)dimscoords)rr'r'r'rr()dimr)rqrrrr dask.utilsrrsshape DataArrayarangegroupbyrrrxr) rrrzrrngroupsarrclimanom anom_meanr~rs rtest_anom_meanrvs  W % %B  X & &B&&&&&& 9  ;;w''1,-   D jmq G ,, c] $*Q- 8 87 BCD   C 9  F6  2 2D ,, c] ! q 012   C ;;u   " "v " . .D ;;u   $D f %%Ii668899KAx x==A      rc idtddfdtddfdtdd fd td d fd tddfdttdddgifdttdddgifdttdddgifd ttdddgifdttdddgifdtdfdtdfdtdfdtdfdtdfdtdfdtdfidtdfdtgdfdtdfdtd fdtdfd tdfdtdd gfdtdfdtdfd tdfd tdfdtdfd!tdfd"tdfd#tdfd$td fd%td ftgd&ftd%gftd'd(gfd)}t|}gd*}|D]*}|||dksJ|||d ksJ+dS)+Nr0rr)r#rr)rrr)r0r'r)r#r'r)rr'rr0r(r)r#r(r)rr(r)r0r)r)r#r)r)rr)rr0r&r)r#r&r)rr&r random_sampler)r1rr)g1r)r1r'r)g3r)b0rr)rr'r)rr(rc0rrrr'rrr(r)rrr)rrr)rr'r)rrr)rr'rrrrrr'rrr(rrr)r)rr&r)rrrrrrrrr'r)rr)g2r)rrrrr5)r7r2(nodes_to_finish_before_loading_more_datans rtest_anom_mean_rawrsF& al3& al3&  al3&  al3 &  al3 &  aOT62rB &  aOT62rB&  aOT62rB&  aOT62rB&  aOT62rB&  a^&  a^&  q+&&  q+&&  q+&&  q,'!& " q,'#& & $ q,'%& & AAAAB'& ( q+&)& * q+&+& , q,'-& . q,'/& 0 A l341& 2 q+&3& 4 q+&5& 6 q+&7& 8 q+&9& : q+&;& < q+&=& > q+&?& @ q+&A& B q+&C& D q+&E& & FQQQR,( [12K& & & CP c A000, 6%%ta n$$$$ta n$$$$$%%rcidtddgfdtgdfdddtddfd td d fd td dfdtddfdtddgfdtgdfdddtddfdtd dfdtddfdtddfdtdfd tdfdtdfidtdfdtdfdtdd gfdtgd!fd d"d#tdfd$td fd%td fd&tdfdtdfd tdfd tdfdtdfd'tdfdttd'dfgfd"ttd'dfgfdttddfgf}id(td)d*gfd)tgd+fd*d,d-td.d/fd0td1d2fd3td4d5fd6td7d8fd9td:d;gfd:tgd<fd;d=d>td/dfd?td2dfd@td5dfdAtd8dfd/tdBdCgfd2tdBdCgfd5tdBdCgfid8tdBdCgfdDtdBdCgfdEtdFdGgfdFtgdHfdGdIdJtd.dfdKtd1dfdLtd4dfdMtd7dfd.tdBdCgfd1tdBdCgfd4tdBdCgfd7tdBdCgfdNtdBdCgfdIttdNdfgfd,ttdNdDfgfd=ttdDdfgf}tt|d}tt|d}||ksJdS)ON)z3mean_agg-aggregate-10d721567ef5a0d6a0e1afae8a87c066rrr)5mean_combine-partial-17c7b5c6eed42e203858b3f6dde16003rrrr)rr'rrr)+mean_chunk-98a32cd9f4fadbed908fffb32e0c9679rrrrrr'rrrrr(rrrrr)rrr)z@mean_chunk-mean_combine-partial-17c7b5c6eed42e203858b3f6dde16003r'rrrr).random_sample-e16bcfb15a013023c98a21e2f03d66a9rrrr).random_sample-02eaa4a8dbb23fac4db22ad034c401b3rrrrr)rr'rrr)rr'rrrr)rr(rrr)rr(rrrr)rr)rrr)rr)rrr)z3mean_agg-aggregate-fdb340546b01334890192fcfa55fa0d9rrr)5mean_combine-partial-23adb4747560e6e33afd63c5bb179709rrrr)rr'rrr)+mean_chunk-7edba1c5a284fcec88b9efdda6c2135frrrrrr'rrrrr(rrrrr)rrr)z@mean_chunk-mean_combine-partial-23adb4747560e6e33afd63c5bb179709r'rrrrr(rrrr')rr&rrr)z3mean_agg-aggregate-cc19342c8116d616fc6573f5d20b5762rrr)5mean_combine-partial-0c98c5a4517f58f8268985e7464daacerrrr)rr'rrr)+mean_chunk-540e88b7d9289f6b5461b95a0787af3errrrrr'rrrrr(rrrrr)rrr)z@mean_chunk-mean_combine-partial-0c98c5a4517f58f8268985e7464daacer'rrrrrrr)rr&rrr)z3mean_agg-aggregate-e79dd3b9757c9fb2ad7ade96f3f6c814rrr)5mean_combine-partial-e7d9fd7c132e12007a4b4f62ce443a75rrrr)rr'rrr)+mean_chunk-0df65d9a6e168673f32082f59f19576arrrrrr'rrrrr(rrrrr)rrr)z@mean_chunk-mean_combine-partial-e7d9fd7c132e12007a4b4f62ce443a75r'rrrr).random_sample-a155d5a37ac5e09ede89c98a3bfcadffrrrr).random_sample-241fdbadc062900adc59d1a79c4c41e1rrrrr)rr'rrr)rr'rrrr)rr(rrr)rr(rrrr)rr)rrr)rr)rrr)z3mean_agg-aggregate-c7647920facf0e557f947b7a6626b7berrr)5mean_combine-partial-57413f0bb18da78db0f689a096c7fbbfrrrr)rr'rrr)+mean_chunk-d6bd425ea61739f1eaa71762fe3bbbd7rrrrr r'rrrr r(rrrr r)rrr)z@mean_chunk-mean_combine-partial-57413f0bb18da78db0f689a096c7fbbfr'rrrrr r r r)rPr'rr)rr&rrr)z3mean_agg-aggregate-05071ebaabb68a64c180f6f443c5c8f4rrr)5mean_combine-partial-a7c475f79a46af4265b189ffdc000bb3rrrr)r r'rrr)+mean_chunk-fd17feaf0728ea7a89d119d3fd172c75rrrrrr'rrrrr(rrrrr)rrr)z@mean_chunk-mean_combine-partial-a7c475f79a46af4265b189ffdc000bb3r'rrrrrrr)rr&rrr)rrr)firstotherfirst_pressuresecond_pressures rtest_flaky_array_reductionrs+l H UU K l  N    P l " NP #l 0 D J JF 1l : D J JF ;l D D J JF El N D J JF Ol X I UU K Yl f N    P gl x NP yl F D J F Gl P D J F Ql Z D J F [l d D J F el n GAol p GAql r GAsl l t GAul v GAwl x I UU K yl F N    P Gl X NP Yl f D JF gl n D JF ol v D JF wl ~ D JF l F GAGl H GAIl J GAKl L GAMl N GAOl P  RR  ]l p  OQR S T }l D  OQR S T Ql l E\b H UU K b  N    P b " NP #b 0 D J JF 1b : D J JF ;b D D J JF Eb N D J JF Ob X I UU K Yb f N    P gb x NP yb F D J F Gb P D J F Qb Z D J F [b d D J F eb n G    I ob z G    I {b F G    I Gb b R G    I Sb ^ G    I _b j I UU K kb x N    P yb J NP Kb X D J F Yb b D J F cb l D J F mb v D J F wb @ G    I Ab L G    I Mb X G    I Yb d G    I eb p G    I qb |  OQR S T Ib P  RR  ]b p  OQR S T }b b EFU++A.//N+e,,Q/00O _ , , , , , ,rc idtdfdtdfdtdfdtdfdtdfdttdfgfdttdfgfd ttdfgfd ttdfgfd dd dd d dd dtd fdtdfdtd fdtd fttddddfgfttddddfgfddtdd ftdd ftddftdd ftddftddfd }t|tfdtdDt fdtdDksJdS)NA0r')A1r)rr')A2r)rr')B1r)rr')B2r')rr)B11r)rr')B22r)rr')C1r)r r')C2r)r!r'Err#r'EErr&r') r$r"r%r')F1r)r(r')r(r()F2r)r)r')r)r(c3,K|]}d|fVdS)r(Nrr!rr2s rrlz&test_flox_reduction..s+00q$}000000rr)c3,K|]}d|fVdS)r)Nrr+s rrlz&test_flox_reduction..s+6X6Xq$}6X6X6X6X6X6Xr)rrrr+rrs @rtest_flox_reductionr-fs  q!f Aq6  Aq6  Aq6   Aq6   AI'(   AI'(  AI'(  AI'(  I  I  I  I  Az?  Az?  Az?! " Az?# $9iIFGH9iIFGHtZ(tZ(tY'tZ(tZ(tY'7   C: c A 0000uQxx000 0 036X6X6X6XuUVxx6X6X6X3X3X X X X X X Xr)F); __future__rrqr dask.baser dask.corer dask.orderrrrdask.utils_testr r fixturer rrr3r8r=r?rArCrErLrXr\rrrrrrrrrrrrrrrrrrrr>rrrmark parametrizerrrrr-rrrr6sv"""""" ((((((8888888888$$$$$$$$)***+*++++   +++"(!(!(!V8!!!@$+$+$+N0$,,,   JJJ( $MMM<---`")")")J888@@@!!!$9!9!9!x":":":J%9%9%9P*$&$&$&N Q Q Q4!4!4!nX@X@X@v6%6%6%r,,,8e5e5e5PVVVr8 5M## #<B5%5%5%pT-T-T-n Y Y Y Y Yr