0Fie<ddlmZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z ddl m Z ddlmZddlmZddlm Z ejeZGd d Zd Z dd Z ddZGddZdS)) annotationsN)Queue)Thread)sleep)merge)gen)timec.eZdZdZdZdZdZdZdZdZ dZ d S) bcolorszzzzzzzzN) __name__ __module__ __qualname__HEADEROKBLUEOKGREENWARNINGFAILENDCBOLD UNDERLINE:lib/python3.11/site-packages/distributed/deploy/old_ssh.pyr r s6 F FGG D D DIIIrr c " ddl}ddlm ddlm}m}|}||d} tj d tj | dddd dd d nA#||f$r4}td tjzd dddztjzttjdzt'|ztjzt)j|dz }|dkr@td tjzdztjzt-jdtdtjzd|dztjzt1dYd}~nd}~wwxYwtddd|ddzdzd\}j d fd fd fd }|} d!rHt1d"| |rnd!Ht?} t?| d#zkrD d$|rn$t1d"t?| d#zkD !|!dS)%Nr) PipeTimeout)PasswordRequiredException SSHExceptionTparamikoaddress ssh_usernamessh_portssh_private_key)hostnameusernameport key_filenamecompresstimeoutbanner_timeoutz[ dask ssh ] : zDSSH connection error when connecting to {addr}:{port} to run '{cmd}'cmd)addrr&r+z, SSH reported this exception: z/SSH connection failed after 3 retries. Exiting.z zRetrying... (attempt z/3)z[ {label} ] : {cmd}label)r/r+z$SHELL -i -c '')get_pty皙?c }t|dkr|}td|ddd|}t|dkdSdS#tjf$rYdSwxYw)z< Read stdout stream, time out if necessary. rzstdout from ssh channel: %s output_queuez[ {label} ] : {output}r/)r/outputN) readlinelenrstriploggerdebugputformatsocketr))linercmd_dictstdouts rread_from_stdoutz#async_ssh..read_from_stdout|s ??$$Dd))a--{{}} :DAAA(,,,33&w/4 ((d))a------V^,    DD sB3B::CCc }t|dkr|}td|dddtjz|ztj z}t|dkdSdS#tj f$rYdSwxYw)z< Read stderr stream, time out if necessary. rzstderr from ssh channel: %sr4[ {label} ] : r/r/N) r6r7r8r9r:r;r<r rrr=r))r>rr?stderrs rread_from_stderrz#async_ssh..read_from_stderrs ??$$Dd))a--{{}} :DAAA(,,$++(72C+DDl#l# ((d))a------V^,    DD sCCC+*C+cHrx}dddtjzdzt |ztjzdSdS)zp Communicate a little bit, without blocking too long. Return True if the command ended. r4rCr/rDz'remote process exited with exit status TN)exit_status_readyrecv_exit_statusr;r<r rstrr) exit_statuschannelr?rFrAs r communicatezasync_ssh..communicates   $ $ & & !2244K ^ $ ( ( ''hw.?'@@,;<k""#,    4  r input_queueg?g@)"rparamiko.buffered_piperparamiko.ssh_exceptionrr SSHClientset_missing_host_key_policy AutoAddPolicylogging getLoggersetLevelWARNconnectprintr rr<rrJ traceback print_excos_exitr exec_commandrL settimeout get_transportempty send_ignorer sendclose)r?rrrsshretriesestdinrM transportstartrrLrFrArEr@s` @@@@@@r async_sshrl"sROOO222222NNNNNNNN     C##H$:$:$<$<===G?>   j ) ) 2 27< @ @ @ KK!),!.1j)%&78!     78. . .  !,!!'!),!*- "("",     @Aa&&,      ! ! ! qLG!||%l#GHl#  !,6'6667,    !HHHHHHHH]. #?P  & &Xg->HUO & T TUUU,,8E?*S0$-E66 nG s$&,!!##I = ! ' ' ) ) c  ;==   = ! ' ' ) ) FFE &&53;   W ;==   c &&53;   MMOOOIIKKKKKsA%CG<D*G77G<c d|p tj|}|#d|d|}|d|||z }tjd|d|tj}t } t } ||||| | |||d } tt| g } d | _ | t| d | iS) Nz8{python} -m distributed.cli.dask_scheduler --port {port})pythonr& mkdir -p  && z,&> {logdir}/dask_scheduler_{addr}:{port}.log)r,r&logdirz scheduler :) r+r/rr&rNr4r r!r"targetargsTthread) r<sys executabler rrrrrldaemonrkr) rqr,r&r r!r" remote_pythonr+r/rNr4r?rvs rstart_schedulerr{s E K K.T L  C +&++c++ =DDDE    | B Bt B Bd BGL B BE''K77L"$$*  H9H: 6 6 6FFM LLNNN Hf- . ..rdistributed.cli.dask_workerc d|dkrdndz}| s|dz }| r|dz }| r|dz }| r|dz }|| p tj||||||| | | }||d | z }|"d |d |}|d||z }d|}t}t}||||||||d}t t |g}d|_|t|d|iS)NzX{python} -m {remote_dask_worker} {scheduler_addr}:{scheduler_port} --nthreads {nthreads}r-z --nworkers {n_workers}z --host {worker_addr}z --memory-limit {memory_limit}z --worker-port {worker_port}z --nanny-port {nanny_port}) rnremote_dask_workerscheduler_addrscheduler_port worker_addrnthreads n_workers memory_limit worker_port nanny_portz$ --local-directory {local_directory})local_directoryrorpz%&> {logdir}/dask_scheduler_{addr}.log)r,rqzworker )r+r/rrNr4r r!r"rsTrv) r<rwrxrrrlryrkr)rqrrrrrr r!r"nohostrrrrzrrr+r/rNr4r?rvs r start_workerrs& @IQ#<#    $k # #E''K77L"$$*  H9H: 6 6 6FFM LLNNN Hf- . ..rceZdZ ddZejdZedZej d Zed Z d Z d Z d Z dZdZdS) SSHClusterrNFr|c ||_||_||_|dd}|r7t dd|||td|tj dt|}n|d}||_ ||_ ||_ ||_| |_| |_| |_| |_||_||_||_ddl}| t.j| d|jd z} t7t8jd | zt8jz| |_ g|_!tE| |||||| |_#g|_$|D]}|%|dS) Nnprocsz.__init__() got an unexpected keyword argument z, z=Both nprocs and n_workers were specified. Use n_workers only.zZThe nprocs argument will be removed in a future release. It has been renamed to n_workers.r-rz dask-ssh_z%Y-%m-%d_%H:%M:%SzaOutput will be redirected to logfiles stored locally on individual worker nodes under "{logdir}".)rq)&rrrpop TypeErrorjoinkeys ValueErrorwarningswarn FutureWarningrr r!r"rrzrrrrrdatetimer]pathnowstrftimerZr rr<rrqthreadsr{ schedulerworkers add_worker)selfrr worker_addrsrrr r!r"rrqrzrrrrrkwargsrrr,s r__init__zSSHCluster.__init__Qs (-,  Hd++  [6;;==AYAY[[   )"7O   M(    II  I"(  . *(&$"4.   W\\h/3355>>?RSSSF #NNTf!OUOO,      )            " "D OOD ! ! ! ! " "rcdSNrrs r_startzSSHCluster._starts rcDtjdt|jSNz[The nprocs attribute will be removed in a future release. It has been renamed to n_workers.rrrrrs rrzSSHCluster.nprocss(  $    ~rcHtjdt||_dSrr)rvalues rrzSSHCluster.nprocss+  $    rc$d|j|jfzS)Nz%s:%d)rrrs rscheduler_addresszSSHCluster.scheduler_addresss$-t/BCCCrc(|jg|jz} |D]]}|dsAt|d|dA^t dp#t $rYdSwxYw)NTr4r2)rrrbrZgetrKeyboardInterruptr all_processesprocesss rmonitor_remote_processesz#SSHCluster.monitor_remote_processess(4<7   ,==G%n5;;===gn599;;<<<&n5;;=== c  !    DD sA1B BBc|jt|j|j|j||j|j|j|j |j |j |j |j |j|j|j|jdSr)rappendrrqrrrrr r!r"rrrrrzrr)rrs rrzSSHCluster.add_workers   ## ! $ ! "'$!       rc|jg|jz}|D]7}|dd|d8dS)NrNshutdownrv)rrr;rrs rrzSSHCluster.shutdowns_(4<7 $ % %G M " & &z 2 2 2 H  " " $ $ $ $ % %rc|Srrrs r __enter__zSSHCluster.__enter__s rc.|dSr)r)rexc_type exc_valuer[s r__exit__zSSHCluster.__exit__s r) rNNrNFNNNNNr|N)r r rrr coroutinerpropertyrsetterrrrrrrrrrrrPs 8#\"\"\"\"| ]  ] X ]]DDXD&   ,%%%rrr)Nr|N) __future__rrUr]r=rwr[rqueuer threadingrr rtlzrtornadordistributed.metricsrVr r9r rlr{rrrrrrs"""""" $$$$$$  8 $ $ kkk^PT'/'/'/'/p4!S/S/S/S/lmmmmmmmmmmr