
    3 d                        d dl mZ d dlmZmZmZ d dlmZ d dlm	Z	m
Z
mZmZmZmZmZ d dlmZ d dlmZ d dlmZ d dlmZmZ d d	lmZ d d
lmZ d dlmZ d dlm Z  d dl!m"Z" d dl#m$Z$  edd          Z% G d d          Z&dS )    )
formatdate)OptionalTypeTypeVar)defer)ConnectErrorConnectionDoneConnectionLostConnectionRefusedErrorDNSLookupErrorTCPTimedOutErrorTimeoutError)ResponseFailed)signals)Crawler)IgnoreRequestNotConfigured)Request)Response)Settings)Spider)StatsCollector)load_objectHttpCacheMiddlewareTVHttpCacheMiddleware)boundc            
          e Zd Zej        eeeeee	e
eef
ZdededdfdZedee         dedefd            Zd	eddfd
Zd	eddfdZded	edee         fdZdeded	edefdZdeded	edee         fdZd	edededee         ddf
dZ dS )r   settingsstatsreturnNc                    |                     d          st           t          |d                   |          | _         t          |d                   |          | _        |                     d          | _        || _        d S )NHTTPCACHE_ENABLEDHTTPCACHE_POLICYHTTPCACHE_STORAGEHTTPCACHE_IGNORE_MISSING)getboolr   r   policystorageignore_missingr   )selfr   r   s      Flib/python3.11/site-packages/scrapy/downloadermiddlewares/httpcache.py__init__zHttpCacheMiddleware.__init__,   s     344 	 ?k(+=">??IIA{8,?#@AA(KK&../IJJ


    clscrawlerc                      | |j         |j                  }|j                            |j        t          j                   |j                            |j        t          j                   |S )N)signal)r   r   r   connectspider_openedspider_closed)r.   r/   os      r+   from_crawlerz HttpCacheMiddleware.from_crawler4   s_     C '-008MNNN8MNNNr-   spiderc                 :    | j                             |           d S N)r(   open_spiderr*   r7   s     r+   r3   z!HttpCacheMiddleware.spider_opened=   s      (((((r-   c                 :    | j                             |           d S r9   )r(   close_spiderr;   s     r+   r4   z!HttpCacheMiddleware.spider_closed@   s    !!&)))))r-   requestc                 &   |j                             dd          rd S | j                            |          sd|j         d<   d S | j                            ||          }|S| j                            d|           | j        r.| j                            d|           t          d|           d S |j
                            d	           | j                            ||          r| j                            d
|           |S ||j         d<   d S )N
dont_cacheFT_dont_cachezhttpcache/missr7   zhttpcache/ignorezIgnored request not in cache: cachedzhttpcache/hitcached_response)metagetr'   should_cache_requestr(   retrieve_responser   	inc_valuer)   r   flagsappendis_cached_response_fresh)r*   r>   r7   cachedresponses       r+   process_requestz#HttpCacheMiddleware.process_requestC   s5   <L%00 	4 {//88 	*.GL'4 77HH 	J  !1& AAA" P
$$%7$GGG#$NW$N$NOOO4 	##H---;//HH 	"J   @@@!! +9&'tr-   responsec                 b   |j                             dd          r|S d|j        v s	d|j         v r|j                             dd            |S d|j        vrt          d          |j        d<   |j                             dd           }|6| j                            d	|
           |                     ||||           |S | j	        
                    |||          r| j                            d|
           |S | j                            d|
           |                     ||||           |S )Nr@   FrC   rA   DateT)usegmtrD   zhttpcache/firsthandrB   zhttpcache/revalidatezhttpcache/invalidate)rE   rF   rJ   popheadersr   r   rI   _cache_responser'   is_cached_response_valid)r*   r>   rO   r7   rM   s        r+   process_responsez$HttpCacheMiddleware.process_responsea   s_    <L%00 	O x~% 	',)F 	L]D111O )) 	?'1'>'>'>HV$ !))*;TBB 	J  !6v FFF  7NKKKO;//'RR 	"J  !7 GGG!!
3FCCCVXwGGGr-   	exceptionc                     |j                             dd           }|3t          || j                  r| j                            d|           |S d S )NrD   zhttpcache/errorrecoveryrB   )rE   rS   
isinstanceDOWNLOAD_EXCEPTIONSr   rI   )r*   r>   rX   r7   rM   s        r+   process_exceptionz%HttpCacheMiddleware.process_exception   sf     !))*;TBB 	"*t/+
 +
 	" J  !:6 JJJ!!tr-   rM   c                     | j                             ||          r:| j                            d|           | j                            |||           d S | j                            d|           d S )Nzhttpcache/storerB   zhttpcache/uncacheable)r'   should_cache_responser   rI   r(   store_response)r*   r7   rO   r>   rM   s        r+   rU   z#HttpCacheMiddleware._cache_response   s}     ;,,Xw?? 	IJ  !26 BBBL''BBBBBJ  !8 HHHHHr-   )!__name__
__module____qualname__r   r   r   r   r	   r   r
   r   r   IOErrorr[   r   r   r,   classmethodr   r   r   r6   r   r3   r4   r   r   r   rN   rW   	Exceptionr\   rU    r-   r+   r   r      s        	 . T     '(3:	   [)F )t ) ) ) )*F *t * * * *w  8HCU    <*2<B	   >		+4	>D		(		 	 	 	II I 	I
 !*I 
I I I I I Ir-   N)'email.utilsr   typingr   r   r   twisted.internetr   twisted.internet.errorr   r	   r
   r   r   r   r   twisted.web.clientr   scrapyr   scrapy.crawlerr   scrapy.exceptionsr   r   scrapy.http.requestr   scrapy.http.responser   scrapy.settingsr   scrapy.spidersr   scrapy.statscollectorsr   scrapy.utils.miscr   r   r   rf   r-   r+   <module>ru      s   " " " " " " * * * * * * * * * * " " " " " "                  . - - - - -       " " " " " " : : : : : : : : ' ' ' ' ' ' ) ) ) ) ) ) $ $ $ $ $ $ ! ! ! ! ! ! 1 1 1 1 1 1 ) ) ) ) ) ) 7?TUUU yI yI yI yI yI yI yI yI yI yIr-   