
    3 d                        d dl Z d dlZd dlZd dlmZ d dlmZmZ d dlm	Z	 d dl
mZ d dlmZ ddgZ	 d dlZe                    d	           n# e$ r Y nw xY w	 d dlZe                    d
           n# e$ r Y nw xY w G d d          ZdS )    N)NotConfigured)ResponseTextResponse)responsetypes)ScrapyDeprecationWarning)gunzip   gzip   deflate   br   zstdc                   B    e Zd ZdZddZed             Zd Zd Zd Z	dS )	HttpCompressionMiddlewarez`This middleware allows compressed (gzip, deflate) traffic to be
    sent/received from web sitesNc                     || _         d S Nstats)selfr   s     Llib/python3.11/site-packages/scrapy/downloadermiddlewares/httpcompression.py__init__z"HttpCompressionMiddleware.__init__    s    


    c                     |j                             d          st          	  | |j                  S # t          $ r5 t          j        dt                      |             }|j        |_        |cY S w xY w)NCOMPRESSION_ENABLEDr   zHttpCompressionMiddleware subclasses must either modify their '__init__' method to support a 'stats' parameter or reimplement the 'from_crawler' method.)settingsgetboolr   r   	TypeErrorwarningswarnr   )clscrawlerresults      r   from_crawlerz&HttpCompressionMiddleware.from_crawler#   s    ''(=>> 	 	3W]++++ 		 		 		M9 )	   SUUF"=FLMMM		s   4 <A32A3c                 l    |j                             dd                    t                               d S )NzAccept-Encodings   , )headers
setdefaultjoinACCEPTED_ENCODINGS)r   requestspiders      r   process_requestz)HttpCompressionMiddleware.process_request4   s.    ""#4ejjAS6T6TUUUUUr   c                 f   |j         dk    r|S t          |t                    r|j                            d          }|r|                                }|                     |j        |                                          }| j	        rF| j	        
                    dt          |          |           | j	        
                    d|           t          j        |j        |j        |          }t          ||          }t!          |t"                    rd |d<    |j        d	i |}|s|j        d= |S )
NHEADzContent-Encodingzhttpcompression/response_bytes)r(   zhttpcompression/response_count)r#   urlbody)r   r-   encoding )method
isinstancer   r#   getlistpop_decoder-   lowerr   	inc_valuelenr   	from_argsr,   dict
issubclassr   replace)	r   r'   responser(   content_encodingr.   decoded_bodyrespclskwargss	            r   process_responsez*HttpCompressionMiddleware.process_response7   s`   >V# 	Oh)) 	='/778JKK =+//11#||HM8>>;K;KLL: J((8L))% )   
 J((8 )    (1$,(,\   '===g|44 . *.F:&+8+55f55' = ();<r   c                    |dk    s|dk    rt          |          }|dk    r@	 t          j        |          }n*# t          j        $ r t          j        |d          }Y nw xY w|dk    rdt          v rt          j        |          }|dk    rUdt          v rLt          j                                        t          j
        |                    }|                                }|S )Nr	   s   x-gzipr
   ir   r   )r   zlib
decompresserrorr&   brotli	zstandardZstdDecompressorstream_readerioBytesIOread)r   r-   r.   readers       r   r4   z!HttpCompressionMiddleware._decodeW   s    w 	 (i"7 	 $<<Dz! 		22t,,: 2 2 2 tS112 u 	+*<!< 	+$T**Dw 	!7.@#@ 	! /11??
4@P@PQQF;;==Ds   8 $AAr   )
__name__
__module____qualname____doc__r   classmethodr!   r)   rA   r4   r/   r   r   r   r      s~        $ $      [ V V V  @    r   r   )rJ   r   rC   scrapy.exceptionsr   scrapy.httpr   r   scrapy.responsetypesr   scrapy.utils.deprecater   scrapy.utils.gzr   r&   rF   appendImportErrorrG   r   r/   r   r   <module>rZ      sT   				   + + + + + + . . . . . . . . . . . . . . ; ; ; ; ; ; " " " " " "z* 	MMMe$$$$ 	 	 	D		g&&&& 	 	 	D	P P P P P P P P P Ps#   A AAA2 2A:9A: