
    3 d7                     `   d Z ddlZddlmZmZ ddlmZ ddlmZm	Z	m
Z
mZmZmZmZmZmZ ddlmZmZ ddlmZ ddlmZmZ dd	lmZ dd
lmZ ddlmZ ddlm Z m!Z! ddl"m#Z# ddl$m%Z%m&Z&m'Z'm(Z( ddl)m*Z*m+Z+  ej,        e-          Z.eeeef         eegef         Z/de0fdZ1 G d de          Z2dS )zS
Spider Middleware manager

See documentation in docs/topics/spider-middleware.rst
    N)isasyncgenfunctioniscoroutine)islice)	AnyAsyncGeneratorAsyncIterableCallable	GeneratorIterableTupleUnioncast)DeferredinlineCallbacks)Failure)RequestSpider)_InvalidOutput)Response)MiddlewareManager)as_async_generatorcollect_asyncgen)build_component_list)deferred_f_from_coro_fdeferred_from_coromaybe_deferred_to_futuremustbe_deferred)MutableAsyncChainMutableChainreturnc                 :    t          | t          t          f          S N)
isinstancer   r   )os    4lib/python3.11/site-packages/scrapy/core/spidermw.py_isiterabler&   ,   s    a(M2333    c                       e Zd ZdZ fdZed             Z fdZdede	de
ded	ef
d
Zde	dedeeef         dedeeef         d	eeef         fdZ	 dde	dededed	eeef         f
dZe	 dde	dedeeef         ded	ef
d            Zde	dedeeef         d	eeef         fdZdede	de
ded	ef
dZded	efdZedede d	ede!e"e!e!f         f         fd            Z# xZ$S )SpiderMiddlewareManagerzspider middlewarec                 B     t                      j        |  d| _        d S )NF)super__init__downgrade_warning_done)selfmiddlewares	__class__s     r%   r,   z SpiderMiddlewareManager.__init__4   s%    +&&&+###r'   c                 F    t          |                    d                    S )NSPIDER_MIDDLEWARES)r   getwithbase)clssettingss     r%   _get_mwlist_from_settingsz1SpiderMiddlewareManager._get_mwlist_from_settings8   s    #H$8$89M$N$NOOOr'   c                    t                                          |           t          |d          r%| j        d                             |j                   t          |d          r%| j        d                             |j                   |                     |d          }| j        d                             |           t          |dd           }| j        d                             |           d S )Nprocess_spider_inputprocess_start_requestsprocess_spider_outputprocess_spider_exception)
r+   _add_middlewarehasattrmethodsappendr8   
appendleftr9   _get_async_method_pairgetattr)r.   mwr:   r;   r0   s       r%   r<   z'SpiderMiddlewareManager._add_middleware<   s    ###2-.. 	QL/0778OPPP2/00 	YL12==b>WXXX $ ; ;B@W X X,-889NOOO#*2/I4#P#P /0;;<TUUUUUr'   scrape_funcresponserequestspiderr    c                 @   | j         d         D ]}t          t          |          }	  |||          }|(|j         dt	          |           }t          |          P# t
          $ r  t          $ r  |t                      ||          cY c S w xY w ||||          S )Nr8   )rE   rG   z- must return None or raise an exception, got )r>   r   r	   __qualname__typer   	Exceptionr   )r.   rD   rE   rF   rG   methodresultmsgs           r%   _process_spider_inputz-SpiderMiddlewareManager._process_spider_inputG   s     l#9: 	? 	?F(F++F?&AAA .!. E E6:6llE E  )---. "    ? ? ?"{799gv>>>>>>>?{8Wf555s   7A-BBiterableexception_processor_index
recover_toc                      dt           f fd}dt          f fd}t          |t                    r ||          S  ||          S )NrP   c              3      K   	 | D ]}|V  d S # t           $ r[}                    t          |                    }t          |t                    r                     |           Y d }~d S d }~ww xY wr"   rK   _process_spider_exceptionr   r#   extend	rP   rexexception_resultrQ   rR   rE   r.   rG   s	       r%   process_syncz@SpiderMiddlewareManager._evaluate_iterable.<locals>.process_syncf   s      	4!  AGGGG  4 4 4#'#A#Afgbkk3L$ $  .88 !!"23333333334s   	 
A5AA00A5c                   K   	 | 2 3 d {V }|W V  6 d S # t           $ r[}                    t          |                    }t          |t                    r                     |           Y d }~d S d }~ww xY wr"   rU   rX   s	       r%   process_asynczASpiderMiddlewareManager._evaluate_iterable.<locals>.process_asyncr   s      	4'       !GGGGG  (xx 4 4 4#'#A#Afgbkk3L$ $  .88 !!"23333333334s     
A<AA77A<)r   r   r#   )r.   rE   rG   rP   rQ   rR   r\   r^   s   ``` ``  r%   _evaluate_iterablez*SpiderMiddlewareManager._evaluate_iterable^   s    
	48 
	4 
	4 
	4 
	4 
	4 
	4 
	4 
	4 
	4 
	4
	4- 
	4 
	4 
	4 
	4 
	4 
	4 
	4 
	4 
	4 
	4 h.. 	+ =***|H%%%r'   r   _failurestart_indexc                    |j         }t          |t                    r|S t          | j        d         |d           }t          ||          D ]\  }}|t          t          |          } ||||          }	t          |	          rE| 	                    |||	|dz             }
|
j
        r	|
j        c S d|j         d}t          |          |	|j         dt          |	           }t          |          |S )Nr;   start)rE   	exceptionrG      zAsync iterable returned from z cannot be downgradedz& must return None or an iterable, got )valuer#   r   r   r>   	enumerater   r	   r&   _process_spider_outputcalledrM   rI   rJ   )r.   rE   rG   r`   ra   re   method_listmethod_indexrL   rM   dfdrN   s               r%   rV   z1SpiderMiddlewareManager._process_spider_exception   sP    N	i00 	OL34k4
 
 %.k$M$M$M 	* 	* L& (F++FVX6RRRF6"" * !% ; ;fflQ.>! !
 : &:%%% af6I```$S))) * * : :+/<<: :  %S)))r'   rM   c           	   #   6  K   t          |t                    }|rt                      }nt                      }t	          | j        d         |d           }t          ||          D ]\  }}	|		dx}
}t          |	t                    r|	\  }}|r|n|}n)|	}|st          |          rd}
n|rt          |          sd}	 |
rt          |          }n|r| j
        s*t                              d|j         d           d| _
        t          |t                    sJ t          t          |                    V }t          |t                    r-t          t          |                    V }t          |          } ||||          }nY# t           $ rL}|                     ||t%          |          |dz             }t          |t$                    r |cY d }~c S d }~ww xY wt'          |          r|                     ||||dz   |          }nVt+          |          r|                                 |j         d	}n|j         d
t/          |           }t1          |          t          |t                    }|rt          ||          S t          ||          S )Nr:   rc   FTzAsync iterable passed to z" was downgraded to a non-async one)rE   rM   rG   rf   z3 must be an asynchronous generator (i.e. use yield)z must return an iterable, got )r#   r   r   r   r   r>   rh   tupler   r   r-   loggerwarningrI   r   r   rK   rV   r   r&   r_   r   closerJ   r   )r.   rE   rG   rM   ra   last_result_is_async	recoveredrk   rl   method_pairneed_upgradeneed_downgrademethod_syncmethod_asyncrL   recovered_collectedrZ   r[   rN   s                      r%   ri   z.SpiderMiddlewareManager._process_spider_output   s\       *&-@@ 	')++II$I T\*ABKQUVV)2;k)R)R)R ;	E ;	E%L+ ,11L>+u-- 	*,7)\)=N;$+ *0B60J0J *#'LL) *2DV2L2L *%)N( F/77FF# F6 ;A8K A A A   7;3%fm<<<<<#56Fv6N6N#O#OOOF!)];; F4F,Y775 5 / /+ %11D$E$E	&PPP ( ( (#'#A#Afgbkk<!3C$ $  .88 '''''''''( 6"" *00fflQ.>	  v&& 
LLNNN!. 6 6 6 C ". * *<<* *  %S)))#-fm#D#D   	8$VY777FI...s   <C
F
G?GGGc                   K   t          |t                    rt                      }nt                      }|                     |||d|          }t          |                     |||                     d {V }t          |t                    rt          ||          S t          |t                    r$t          |           d {V }t          |          }t          ||          S )Nr   )r#   r   r   r   r_   r   ri   r   )r.   rE   rG   rM   rt   rz   s         r%   _process_callback_outputz0SpiderMiddlewareManager._process_callback_output  s      fm,, 	')++II$I((661iPP/''&&AA
 
 
 
 
 
 
 
 fm,, 	8$VY777i// 	:(8(C(C"C"C"C"C"C"C$%899IFI...r'   c                 H    dt           t          t          f         dt           t          t          f         f fd}dt
          dt           t
          t          f         f fd}t           j        ||          }|                    t          |          |           |S )NrM   r    c                 B   K                        |            d {V S r"   )r|   )rM   rE   r.   rG   s    r%   process_callback_outputzHSpiderMiddlewareManager.scrape_response.<locals>.process_callback_output%  s3       66xPPPPPPPPPr'   r`   c                 2                         |           S r"   )rV   )r`   rE   r.   rG   s    r%   r;   zISpiderMiddlewareManager.scrape_response.<locals>.process_spider_exception*  s    11(FHMMMr'   )callbackerrback)
r   r   r   r   r   r   r   rO   addCallbacksr   )r.   rD   rE   rF   rG   r   r;   rm   s   ` ` `   r%   scrape_responsez'SpiderMiddlewareManager.scrape_response  s    	Q(M12	Q<!223	Q 	Q 	Q 	Q 	Q 	Q 	Q 	Q
	Nw 	N5,AV;W 	N 	N 	N 	N 	N 	N 	N 	N &Xw
 
 	+,CDD, 	 	
 	
 	
 
r'   c                 0    |                      d||          S )Nr9   )_process_chain)r.   start_requestsrG   s      r%   r9   z.SpiderMiddlewareManager.process_start_requests6  s    ""#;^VTTTr'   rC   
methodnameNc           	         t          | |d           }|dz   }t          | |d           }|s|S |s+t                              d| j         d| d| d           d S t	          |          s$t                              |j         d           |S t	          |          r't                              |j         d| d           d S ||fS )	N_asynczMiddleware z has z	 without z, skipping this method.z: is not an async generator function, skipping this method.z& is an async generator function while z exists, skipping both methods.)rB   rp   errorrI   r   )rC   r   normal_methodmethodname_asyncasync_methods        r%   rA   z.SpiderMiddlewareManager._get_async_method_pair:  s7     J55%0r#3T:: 	!   	LL?bo ? ?4D ? ?%? ? ?   4!,// 	!LL, F F F   ! m,, 	LL - * *,<* * *  
 4l**r'   )r   )%__name__
__module__rI   component_namer,   classmethodr6   r<   
ScrapeFuncr   r   r   r   rO   r   r   r   intr   r   r
   r   r_   r   rV   r   r   ri   r|   r   r9   staticmethodstrr	   r   rA   __classcell__)r0   s   @r%   r)   r)   0   s       (N, , , , , P P [P	V 	V 	V 	V 	V66 6 	6
 6 
6 6 6 6."&"& "& -/0	"&
 $'"& ,(99:"& 
y.(	)"& "& "& "&R * ** * 	*
 * 
w$	%* * * *^  W/ W/W/ W/ h-.	W/
 W/ 
W/ W/ W/ _W/r/ /*0/:?-@W:X/	|..	// / / /&  	
  
   0UV U U U U U ++ +	tXuXx%788	9+ + + \+ + + + +r'   r)   )3__doc__logginginspectr   r   	itertoolsr   typingr   r   r   r	   r
   r   r   r   r   twisted.internet.deferr   r   twisted.python.failurer   scrapyr   r   scrapy.exceptionsr   scrapy.httpr   scrapy.middlewarer   scrapy.utils.asyncgenr   r   scrapy.utils.confr   scrapy.utils.deferr   r   r   r   scrapy.utils.pythonr   r   	getLoggerr   rp   r   boolr&   r)    r'   r%   <module>r      s;   
  3 3 3 3 3 3 3 3      
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 = < < < < < < < * * * * * * " " " " " " " " , , , , , ,             / / / / / / F F F F F F F F 2 2 2 2 2 2            @ ? ? ? ? ? ? ?		8	$	$ uXw./&A3FG
4d 4 4 4 4f+ f+ f+ f+ f+/ f+ f+ f+ f+ f+r'   