
    3 d                         d Z ddlZddlZddlZddlmZ ddlmZ ddlm	Z	 ddl
mZ ddlmZ ddlmZ dd	lmZ  ej        e          Z G d
 d          ZdS )zH
MemoryUsage extension

See documentation in docs/topics/extensions.rst
    N)import_module)pformat)task)signals)NotConfigured)
MailSender)get_engine_statusc                   T    e Zd Zd Zed             Zd Zd Zd Zd Z	d Z
d Zd	 Zd
S )MemoryUsagec                    |j                             d          st          	 t          d          | _        n# t
          $ r t          w xY w|| _        d| _        |j                             d          | _	        |j         
                    d          dz  dz  | _        |j         
                    d          dz  dz  | _        |j                             d          | _        t          j        |j                   | _        |j                            | j        t$          j        	           |j                            | j        t$          j        	           d S )
NMEMUSAGE_ENABLEDresourceFMEMUSAGE_NOTIFY_MAILMEMUSAGE_LIMIT_MB   MEMUSAGE_WARNING_MBMEMUSAGE_CHECK_INTERVAL_SECONDS)signal)settingsgetboolr   r   r   ImportErrorcrawlerwarnedgetlistnotify_mailsgetintlimitwarninggetfloatcheck_intervalr   from_settingsmailr   connectengine_startedengine_stopped)selfr   s     :lib/python3.11/site-packages/scrapy/extensions/memusage.py__init__zMemoryUsage.__init__   sF   ''(:;; 	 	 )*55DMM 	  	  	 	  #,445KLL%,,-@AADH4O
'../DEELtS%.77-
 
 ,W-=>>	 3G<RSSS 3G<RSSSSSs	   8 A
c                      | |          S N )clsr   s     r'   from_crawlerzMemoryUsage.from_crawler,   s    s7||    c                     | j                             | j         j                  j        }t          j        dk    r|dz  }|S )Ndarwinr   )r   	getrusageRUSAGE_SELF	ru_maxrsssysplatform)r&   sizes     r'   get_virtual_sizezMemoryUsage.get_virtual_size0   s<    }&&t}'@AAK<8# 	DLDr.   c                 r   | j         j                            d|                                            g | _        t          j        | j                  }| j                            |           |	                    | j
        d           | j        rOt          j        | j                  }| j                            |           |	                    | j
        d           | j        rQt          j        | j                  }| j                            |           |	                    | j
        d           d S d S )Nmemusage/startupT)now)r   stats	set_valuer7   tasksr   LoopingCallupdateappendstartr    r   _check_limitr   _check_warningr&   tsks     r'   r$   zMemoryUsage.engine_started7   s   $$%79N9N9P9PQQQ
t{++
#		$%4	000: 	5"4#455CJc"""IId)tI444< 	5"4#677CJc"""IId)tI44444	5 	5r.   c                 P    | j         D ]}|j        r|                                 d S r*   )r=   runningstoprD   s     r'   r%   zMemoryUsage.engine_stoppedF   s5    : 	 	C{ 


	 	r.   c                 j    | j         j                            d|                                            d S )Nmemusage/max)r   r;   	max_valuer7   )r&   s    r'   r?   zMemoryUsage.updateK   s/    $$^T5J5J5L5LMMMMMr.   c                    |                                  }|| j        k    r| j        j                            dd           | j        dz  dz  }t
                              dd|id| j        i           | j        rf| j        j        d          d	| d
t          j
                     }|                     | j        |           | j        j                            dd           | j        j        j        1| j        j                            | j        j        j        d           d S | j                                         d S t
                              dd|dz  dz  i           d S )Nzmemusage/limit_reached   r   z>Memory usage exceeded %(memusage)dMiB. Shutting down Scrapy...memusager   extraBOT_NAMEz# terminated: memory usage exceeded MiB at zmemusage/limit_notifiedmemusage_exceededz'Peak memory usage is %(virtualsize)dMiBvirtualsize)r7   r   r   r;   r<   loggererrorr   r   socketgethostname_send_reportenginespiderclose_spiderrH   info)r&   peak_mem_usagememsubjs       r'   rB   zMemoryUsage._check_limitN   s   ..00DJ& 	L(()A1EEE*t#d*CLLPS! $,/    
   K|,Z8 P P-0P P9?9K9M9MP P  !!$"3T:::",,-FJJJ|") $#00L'.0C     !!#####KK9 5 <=    r.   c                    | j         rd S |                                 | j        k    r| j        j                            dd           | j        dz  dz  }t                              dd|id| j        i           | j        rf| j        j        d          d	| d
t          j
                     }|                     | j        |           | j        j                            dd           d| _         d S d S )Nzmemusage/warning_reachedrM   r   z$Memory usage reached %(memusage)dMiBrN   r   rO   rQ   z warning: memory usage reached rR   zmemusage/warning_notifiedT)r   r7   r   r   r;   r<   rU   r   r   rW   rX   rY   )r&   r_   r`   s      r'   rC   zMemoryUsage._check_warningl   s(   ; 	F  ""T\1 	L(()CQGGG,%,CNN6S! $,/    
   M|,Z8 O O,/O O8>8J8L8LO O  !!$"3T:::",,-H!LLLDKKK	 	r.   c                    | j         j        }d|                    d          dz  dz   d}|d|                    d          dz  dz   dz  }|d|                                 dz  dz   dz  }|dz  }|d	z  }|t	          t          | j         j                            z  }|d	z  }| j                            |||           d
S )z7send notification mail with some additional useful infoz!Memory usage at engine startup : r9   r   zM
z Maximum memory usage          : rJ   z Current memory usage          : zHENGINE STATUS ------------------------------------------------------- 
z
N)	r   r;   	get_valuer7   r   r	   rZ   r"   send)r&   rcptssubjectr;   ss        r'   rY   zMemoryUsage._send_report   s    "d@R0S0STX0XY]0]ddd	`0O0OPT0TUY0Y````	X0E0E0G0G0LT0QXXXX	X	
 	
V	W&t|':;;<<<	V	ugq)))))r.   N)__name__
__module____qualname__r(   classmethodr-   r7   r$   r%   r?   rB   rC   rY   r+   r.   r'   r   r      s        T T T*   [  5 5 5  
N N N  <  (* * * * *r.   r   )__doc__loggingrW   r4   	importlibr   pprintr   twisted.internetr   scrapyr   scrapy.exceptionsr   scrapy.mailr   scrapy.utils.enginer	   	getLoggerrh   rU   r   r+   r.   r'   <module>rv      s    
   



 # # # # # #       ! ! ! ! ! !       + + + + + + " " " " " " 1 1 1 1 1 1		8	$	$w* w* w* w* w* w* w* w* w* w*r.   