
    3 d(                         d dl Z d dlZd dlmZ d dlmZ d dlmZ d dlm	Z	m
Z
 d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZmZ d dlmZ d dlmZ d dlmZ  ej        e          Zd Z G d d          Z dS )    N)defaultdict)	signature)warn)DeferredDeferredList)Failure)NO_CALLBACK)Settings)SequenceExclude)defer_resultmustbe_deferred)ScrapyDeprecationWarning)failure_to_exc_info)arg_to_iterc                     | S N )responses    6lib/python3.11/site-packages/scrapy/pipelines/media.py_DUMMY_CALLBACKr      s    O    c                       e Zd ZdZ G d d          ZddZd ZddZed             Z	d	 Z
d
 Zd Zd Zd Zd Zd Zd Zd ZdddZd ZdddZd Zd ZddddZdS )MediaPipelineTc                       e Zd Zd ZdS )MediaPipeline.SpiderInfoc                 z    || _         t                      | _        i | _        t	          t
                    | _        d S r   )spidersetdownloading
downloadedr   listwaitingselfr   s     r   __init__z!MediaPipeline.SpiderInfo.__init__   s/     DK"uuD DO&t,,DLLLr   N)__name__
__module____qualname__r%   r   r   r   
SpiderInfor      s#        	- 	- 	- 	- 	-r   r)   Nc                 J   || _         i | _        t          |t                    s|t	          |          }t          j        | j        d|          }|                     |d          d          | _	        | 
                    | j	                   |                                  d S )Nr   )base_class_namesettingsMEDIA_ALLOW_REDIRECTSF)download_func_expects_item
isinstancedictr
   	functoolspartial_key_for_pipegetboolallow_redirects_handle_statuses_make_compatible)r$   r.   r,   resolves       r   r%   zMediaPipeline.__init__$   s    *h%% 	* 	*))H#(
 
 
  (//8O0P0PRWXXd2333 	r   c                 `    d | _         |r$t          t          dd                    | _         d S d S )Ni,  i  )handle_httpstatus_listr   range)r$   r6   s     r   r7   zMediaPipeline._handle_statuses3   s=    &*# 	K*9%S//*J*JD'''	K 	Kr   c                     | j         j        }|                                 d| }|r||k    s|r|                    |          s|S |S )z
        >>> MediaPipeline()._key_for_pipe("IMAGES")
        'IMAGES'
        >>> class MyPipe(MediaPipeline):
        ...     pass
        >>> MyPipe()._key_for_pipe("IMAGES", base_class_name="MediaPipeline")
        'MYPIPE_IMAGES'
        _)	__class__r&   upperget)r$   keyr+   r,   
class_nameformatted_keys         r   r4   zMediaPipeline._key_for_pipe8   so     ^,
%++--5555	_,	 	 LL//		 Jr   c                     	 |                      |j                  }n# t          $ r  |             }Y nw xY w||_        |j        |_        |S r   )from_settingsr,   AttributeErrorcrawlerrequest_fingerprinter_fingerprinter)clsrH   pipes      r   from_crawlerzMediaPipeline.from_crawlerL   s`    	$$W%566DD 	 	 	355DDD	%;s    44c                 :    |                      |          | _        d S r   )r)   
spiderinfor#   s     r   open_spiderzMediaPipeline.open_spiderV   s    //&11r   c                       j         t                                                   } fd|D             }t          |d          }|                     j                  S )Nc                 >    g | ]}                     |          S r   )_process_request).0rinfoitemr$   s     r   
<listcomp>z.MediaPipeline.process_item.<locals>.<listcomp>\   s+    HHH!&&q$55HHHr   T)consumeErrors)rO   r   get_media_requestsr   addCallbackitem_completed)r$   rW   r   requestsdlistdfdrV   s   ``    @r   process_itemzMediaPipeline.process_itemY   ss    t66tTBBCCHHHHHHxHHH5555t2D$???r   c                    | j                             |          }|j        r|j        t          u rt          }n|j        }|j        }t          |_        d |_        |j        v r.t          j        |                                       ||          S t                                          ||          j
        |                                        |j        v rS j                            |           t          | j        ||          }|                    | j        ||           |                    | j        |           |                    fd           |                    fd          S )NrW   c                 p    t                               | j        t          |           dj        i          S )Nr   exc_infoextra)loggererrorvaluer   r   )frV   s    r   <lambda>z0MediaPipeline._process_request.<locals>.<lambda>|   s3    fll"5a"8"84;@W #   r   c                     S r   r   )r>   wads    r   rk   z0MediaPipeline._process_request.<locals>.<lambda>   s    S r   )rJ   fingerprintcallbackr	   r   errbackr    r   addCallbacksr   r"   appendr   addr   media_to_downloadr[   _check_media_to_downloadaddBoth!_cache_result_and_execute_waiters
addErrback)	r$   requestrV   rW   fpcbebr_   rm   s	     `     @r   rS   zMediaPipeline._process_request`   s    ,,W55 	"7#3{#B 	" BB!B_&   	J 344AA"bIII jj%%b"--R$$$ !! 	J 	R   d4gt$OOO5w4PPPD:BEEE   	
 	
 	

 {{====)))r   c                     g d}|D ]F}t          | |d          }t          |          r$t          | ||                     |                     GdS )zMMake overridable methods of MediaPipeline and subclasses backwards compatible)	file_path
thumb_pathrt   media_downloadedfile_downloadedimage_downloaded
get_imagesN)getattrcallablesetattr_compatible)r$   methodsmethod_namemethods       r   r8   zMediaPipeline._make_compatible   su    
 
 
 # 	E 	EKT;55F Ek4+;+;F+C+CDDD	E 	Er   c                 t                                      t          j                   fd            }|S )z@Wrapper for overridable methods to allow backwards compatibilityc                  t    j         j                 r | i |S |                    dd             | i |S )NrW   )r/   r&   pop)argskwargsfuncr$   s     r   wrapperz*MediaPipeline._compatible.<locals>.wrapper   sS    !$-0 -tT,V,,,JJvt$$$4((((r   )_check_signaturer2   wraps)r$   r   r   s   `` r   r   zMediaPipeline._compatible   sR    d###				) 	) 	) 	) 	) 
		) r   c           
         t          |          }d| j        |j        <   d|j        vrZt	          |          dd         }|dz   }t          |j         d| d|j         d| dt          d	
           d| j        |j        <   d S d S )NTrW      z, *, item=Nonez(self, z) is deprecated, please use )   )
stacklevelF)r   r/   r&   
parametersstrr   r   )r$   r   sig
old_params
new_paramss        r   r   zMediaPipeline._check_signature   s    oo,04=)' 		6S!B$J#&66J= B B B B"mB B4>B B B(	    16Dt}---		6 		6r   c                 J    | j         r| j         |j        d<   d S d|j        d<   d S )Nr;   Thandle_httpstatus_all)r;   meta)r$   ry   s     r   _modify_media_requestz#MediaPipeline._modify_media_request   s6    & 	9595PGL122248GL0111r   c                 d   ||S | j         rFt          | j         ||j                  }|                    | j        ||fd|i| j        ||f           n^|                     |           | j        j        	                    |          }|                    | j        ||fd|i| j        ||f           |S )NrW   )ro   callbackArgscallbackKeywordsrp   errbackArgs)
r.   r   r   rq   r   media_failedr   rH   enginedownload)r$   resultry   rV   rW   r_   s         r   ru   z&MediaPipeline._check_media_to_download   s     	M 	!$"4gt{KKC.%t_"($)$dO      &&w///,%..w77C.%t_"($)$dO     
r   c                    t          |t                    rc|                                 g |_        d |_        t          |j        dd           }t          |t                    rt          |j        dd            |j	        
                    |           ||j        |<   |j                            |          D ]$}t          |                              |           %d S )N__context__)r0   r   cleanFailureframesstackr   ri   StopIterationr   r   remover    r"   r   r   chainDeferred)r$   r   rz   rV   contextrm   s         r   rw   z/MediaPipeline._cache_result_and_execute_waiters   s    fg&& 	;!!!FMFL. flM4@@G'=11 ;mT:::###$<##B'' 	4 	4C  ..s3333	4 	4r   rb   c                    dS )z&Check request before starting downloadNr   )r$   ry   rV   rW   s       r   rt   zMediaPipeline.media_to_download       r   c                     dS )z&Returns the media requests to downloadNr   )r$   rW   rV   s      r   rZ   z MediaPipeline.get_media_requests   r   r   c                    |S )zHandler for success downloadsr   )r$   r   ry   rV   rW   s        r   r   zMediaPipeline.media_downloaded   s    r   c                     |S )zHandler for failed downloadsr   )r$   failurery   rV   s       r   r   zMediaPipeline.media_failed  s    r   c                     | j         rI|D ]F\  }}|s?t                              d| j        j        |dt          |          d|j        i           G|S )z:Called per item when all media requests has been processedz*%(class)s found errors processing %(item)s)classrW   r   rd   )LOG_FAILED_RESULTSrg   rh   r?   r&   r   r   )r$   resultsrW   rV   okri   s         r   r\   zMediaPipeline.item_completed  ss    " 	$  	E LLD"&."94HH!4U!;!;'5	 !    r   c                    dS )z8Returns the path where downloaded media should be storedNr   )r$   ry   r   rV   rW   s        r   r~   zMediaPipeline.file_path  r   r   )NN)r&   r'   r(   r   r)   r%   r7   r4   classmethodrM   rP   r`   rS   r8   r   r   r   ru   rw   rt   rZ   r   r   r\   r~   r   r   r   r   r      s       - - - - - - - -       K K K
   (   [2 2 2@ @ @ *  *  *DE E E"  6 6 69 9 9  2#4 #4 #4L 8<        AE         4       r   r   )!r2   loggingcollectionsr   inspectr   warningsr   twisted.internet.deferr   r   twisted.python.failurer   scrapy.http.requestr	   scrapy.settingsr
   scrapy.utils.datatypesr   scrapy.utils.deferr   r   scrapy.utils.deprecater   scrapy.utils.logr   scrapy.utils.miscr   	getLoggerr&   rg   r   r   r   r   r   <module>r      s[        # # # # # #             9 9 9 9 9 9 9 9 * * * * * * + + + + + + $ $ $ $ $ $ 2 2 2 2 2 2 < < < < < < < < ; ; ; ; ; ; 0 0 0 0 0 0 ) ) ) ) ) )		8	$	$  { { { { { { { { { {r   