
    3 d"                         d Z ddlZddlZddlZddlmZ ddlmZ ddlm	Z	 ddl
mZmZmZ ddlmZ ddlmZ dd	lmZmZ dd
lmZ ddlmZ ddlmZmZ  G d de          Z G d de          Z G d de          ZdS )zA
Images Pipeline

See documentation in topics/media-pipeline.rst
    N)suppress)BytesIO)ItemAdapter)DropItemNotConfiguredScrapyDeprecationWarning)Request)NO_CALLBACK)FileExceptionFilesPipeline)Settings)md5sum)get_func_argsto_bytesc                   "     e Zd ZdZ fdZ xZS )NoimagesDropz Product with no images exceptionc                 r    t          j        dt          d            t                      j        |i | d S )Nz$The NoimagesDrop class is deprecated   category
stacklevel)warningswarnr   super__init__)selfargskwargs	__class__s      7lib/python3.11/site-packages/scrapy/pipelines/images.pyr   zNoimagesDrop.__init__   sI    2-	
 	
 	
 	

 	$)&)))))    )__name__
__module____qualname____doc__r   __classcell__r   s   @r    r   r      s>        *** * * * * * * * *r!   r   c                       e Zd ZdZdS )ImageExceptionzGeneral image error exceptionN)r"   r#   r$   r%    r!   r    r)   r)   %   s        ''''r!   r)   c                        e Zd ZdZdZdZdZdZi ZdZ	dZ
d fd	Zed	             Zdd
dZdd
dZdd
dZddZd Zd Zddd
dZddd
dZ xZS )ImagesPipelinezEAbstract pipeline that implement the image thumbnail generation logicimager   Z   
image_urlsimagesNc                    	 ddl m} || _        n# t          $ r t	          d          w xY wt                                          |||           t          |t                    s|t          |          }t          j        | j        d|          }|                     |d          | j                  | _        t!          | d          s| j        | _        t!          | d	          s| j        | _        |                     |d	          | j                  | _        |                     |d          | j                  | _        |                     |d
          | j                  | _        |                     |d          | j                  | _        |                     |d          | j                  | _        d | _        d S )Nr   )Imagez8ImagesPipeline requires installing Pillow 4.0.0 or later)settingsdownload_funcr,   )base_class_namer3   IMAGES_EXPIRESIMAGES_RESULT_FIELDIMAGES_URLS_FIELDIMAGES_MIN_WIDTHIMAGES_MIN_HEIGHTIMAGES_THUMBS)PILr2   _ImageImportErrorr   r   r   
isinstancedictr   	functoolspartial_key_for_pipegetintEXPIRESexpireshasattrDEFAULT_IMAGES_RESULT_FIELDr7   DEFAULT_IMAGES_URLS_FIELDr8   getimages_urls_fieldimages_result_field	MIN_WIDTH	min_width
MIN_HEIGHT
min_heightTHUMBSthumbs_deprecated_convert_image)r   	store_urir4   r3   r2   resolver   s         r    r   zImagesPipeline.__init__7   s   	!!!!!!DKK 	 	 	J  	
 	X]SSSh%% 	* 	*))H#,
 
 

  ww/?'@'@$,OOt233 	H'+'GD$t011 	D%)%CD"!)G'(($*@"
 "
 $,<<G)**D,D$
 $
  "1C)D)DdnUU"//''2E*F*FXXll77?#;#;T[II)-&&&s    +c                    | j         d         }|d         |_        |d         |_        |d         |_        |d         |_        |d         |_        |d         |_        |d         |_        |d	         |_        | j         d
         }|d         |_	        |d         pd |_        | j         d         }|d         |_
        |d         |_        |                    d          |_        |d         } | ||          S )Ns3AWS_ACCESS_KEY_IDAWS_SECRET_ACCESS_KEYAWS_SESSION_TOKENAWS_ENDPOINT_URLAWS_REGION_NAMEAWS_USE_SSL
AWS_VERIFYIMAGES_STORE_S3_ACLgsGCS_PROJECT_IDIMAGES_STORE_GCS_ACLftpFTP_USERFTP_PASSWORDFEED_STORAGE_FTP_ACTIVEIMAGES_STORE)r3   )STORE_SCHEMESrX   rY   rZ   r[   r\   r]   r^   POLICYra   FTP_USERNAMEre   getboolUSE_ACTIVE_MODE)clsr3   s3store	gcs_store	ftp_storerT   s         r    from_settingszImagesPipeline.from_settings^   s   #D)$,-@$A!(01H(I%$,-@$A!#+,>#? "*+<"=&}5%l3!"78%d+	#+,<#=	 #$:;Ct	%e,	!)*!5	!).!9	$,$4$45N$O$O	!^,	s9x0000r!   itemc                4    |                      ||||          S )Nrr   )image_downloaded)r   responserequestinfors   s        r    file_downloadedzImagesPipeline.file_downloadedv   s    $$Xw4$HHHr!   c          	          d }|                      ||||          D ]Z\  }}}|$|                    d           t          |          }|j        \  }	}
| j                            ||||	|
dddi           [|S )Nrr   r   )widthheightzContent-Typez
image/jpeg)metaheaders)
get_imagesseekr   sizestorepersist_file)r   rv   rw   rx   rs   checksumpathr-   bufr{   r|   s              r    ru   zImagesPipeline.image_downloadedy   s     $'4d S S 	 	D% '!#;;!JME6J##$77'6 $     r!   c             #   ^  K   |                      ||||          }| j                            t          |j                            }|j        \  }}|| j        k     s|| j        k     r&t          d| d| d| j         d| j         d	          | j	        Jdt          | j                  v| _	        | j	        r(t          j        | j        j         dt                      | j	        r|                     |          \  }	}
n,|                     |t          |j                  	          \  }	}
||	|
fV  | j                                        D ]a\  }}|                     |||||          }| j	        r|                     |	|          \  }}n|                     |	||
          \  }}|||fV  bd S )
N)rv   rx   rs   zImage too small (xz < )response_bodyzq.convert_image() method overridden in a deprecated way, overridden method does not accept response_body argument.)r   )r   )	file_pathr=   openr   bodyr   rN   rP   r)   rS   r   convert_imager   r   r   r"   r   rR   items
thumb_path)r   rv   rw   rx   rs   r   
orig_imager{   r|   r-   r   thumb_idr   r   thumb_image	thumb_bufs                   r    r   zImagesPipeline.get_images   s=     ~~gt$~OO[%%ghm&<&<==
"v4>! 	Vdo%= 	 77 7"7 7>7 7$(O7 7 7   ) 		-<M"E E .D* - ~. P P P5    ) 	++J77JE33++'(-*@*@ ,  JE3 E3"k//11 	5 	5NHdH4d )  J - N)-););E4)H)H&YY)-););E4)M)M&Yk944444	5 	5r!   c                 ^   |)t          j        | j        j         dt          d           |j        dv rX|j        dk    rM| j                            d|j	        d          }|
                    ||           |                    d          }n|j        dk    rb|                    d          }| j                            d|j	        d          }|
                    ||           |                    d          }n |j        dk    r|                    d          }|rZ|                                }	 | j        j        j        }n# t          $ r | j        j        }Y nw xY w|                    ||           n||j        d	k    r||fS t%                      }|                    |d	           ||fS )
Nza.convert_image() method called in a deprecated way, method called without response_body argument.r   r   )PNGWEBPRGBA)   r   r   RGBPJPEG)r   r   r   r"   r   formatmoder=   newr   pasteconvertcopy
ResamplingLANCZOSAttributeError	ANTIALIAS	thumbnailr   save)r   r-   r   r   
backgroundresampling_filterr   s          r    r   zImagesPipeline.convert_image   s    	M>* @ @ @1	    <?* 
	)uzV/C 
	)_MMJUE***&&u--EEZ3 	)MM&))E_MMJUE***&&u--EEZ5  	)MM%((E 	(JJLLE: %)K$:$B!!! : : :$(K$9!!!:OOD"34444 	(5<6+A 	(-''ii

3czs   1E EEc                 j    t          |                              | j        g           }d |D             S )Nc                 :    g | ]}t          |t                     S ))callback)r	   r
   ).0us     r    
<listcomp>z5ImagesPipeline.get_media_requests.<locals>.<listcomp>   s%    ???QK000???r!   )r   rJ   rK   )r   rs   rx   urlss       r    get_media_requestsz!ImagesPipeline.get_media_requests   s6    4  $$T%;R@@??$????r!   c                     t          t                    5  d |D             t          |          | j        <   d d d            n# 1 swxY w Y   |S )Nc                     g | ]	\  }}||
S r*   r*   )r   okr   s      r    r   z1ImagesPipeline.item_completed.<locals>.<listcomp>   s"    :X:X:XQUW:X1:X:X:Xr!   )r   KeyErrorr   rL   )r   resultsrs   rx   s       r    item_completedzImagesPipeline.item_completed   s    h 	Y 	Y:X:X':X:X:XKd67	Y 	Y 	Y 	Y 	Y 	Y 	Y 	Y 	Y 	Y 	Y 	Y 	Y 	Y 	Ys   "AA
Ac                ~    t          j        t          |j                                                            }d| dS )Nzfull/.jpghashlibsha1r   url	hexdigest)r   rw   rv   rx   rs   
image_guids         r    r   zImagesPipeline.file_path   s8    \(7;"7"788BBDD
'z''''r!   c                    t          j        t          |j                                                            }d| d| dS )Nzthumbs//r   r   )r   rw   r   rv   rx   rs   
thumb_guids          r    r   zImagesPipeline.thumb_path   s@    \(7;"7"788BBDD
444J4444r!   )NN)r"   r#   r$   r%   
MEDIA_NAMErM   rO   rE   rQ   rI   rH   r   classmethodrq   ry   ru   r   r   r   r   r   r   r&   r'   s   @r    r,   r,   )   s`       OOJ IJGF ,"*%. %. %. %. %. %.N 1 1 [1. @D I I I I I AE       ;? '5 '5 '5 '5 '5R$ $ $ $L@ @ @  
(4 ( ( ( ( (5d 5 5 5 5 5 5 5 5 5r!   r,   )r%   rA   r   r   
contextlibr   ior   itemadapterr   scrapy.exceptionsr   r   r   scrapy.httpr	   scrapy.http.requestr
   scrapy.pipelines.filesr   r   scrapy.settingsr   scrapy.utils.miscr   scrapy.utils.pythonr   r   r   r)   r,   r*   r!   r    <module>r      s   
                   # # # # # # O O O O O O O O O O       + + + + + + ? ? ? ? ? ? ? ? % $ $ $ $ $ $ $ $ $ $ $ 7 7 7 7 7 7 7 7	* 	* 	* 	* 	*8 	* 	* 	*( ( ( ( (] ( ( (~5 ~5 ~5 ~5 ~5] ~5 ~5 ~5 ~5 ~5r!   