
    3 d                          d dl Z d dlZd dlmZ  ej        e          Zd Z G d d          Z G d d          Z	 G d d	          Z
dS )
    N)create_instancec                     d                     d | D                       }t          j        |                     d                                                    }d                     ||g          S )a!  
    Return a filesystem-safe version of a string ``text``

    >>> _path_safe('simple.org').startswith('simple.org')
    True
    >>> _path_safe('dash-underscore_.org').startswith('dash-underscore_.org')
    True
    >>> _path_safe('some@symbol?').startswith('some_symbol_')
    True
     c                 F    g | ]}|                                 s|d v r|ndS )z-.__)isalnum).0cs     .lib/python3.11/site-packages/scrapy/pqueues.py
<listcomp>z_path_safe.<locals>.<listcomp>   s2    SSS!))++DeDQQSSS    utf8-)joinhashlibmd5encode	hexdigest)textpathable_slotunique_slots      r   
_path_safer   	   sd     GGSSdSSSTTM +dkk&1122<<>>K88]K0111r   c                   b    e Zd ZdZedd            ZddZd Zd Zd Z	d Z
d	 Zd
 Zd Zd ZdS )ScrapyPriorityQueuea  A priority queue implemented using multiple internal queues (typically,
    FIFO queues). It uses one internal queue for each priority value. The internal
    queue must implement the following methods:

        * push(obj)
        * pop()
        * close()
        * __len__()

    Optionally, the queue could provide a ``peek`` method, that should return the
    next object to be returned by ``pop``, but without removing it from the queue.

    ``__init__`` method of ScrapyPriorityQueue receives a downstream_queue_cls
    argument, which is a class used to instantiate a new (internal) queue when
    a new priority is allocated.

    Only integer priorities should be used. Lower numbers are higher
    priorities.

    startprios is a sequence of priorities to start with. If the queue was
    previously closed leaving some priority buckets non-empty, those priorities
    should be passed in startprios.

     c                      | ||||          S Nr   clscrawlerdownstream_queue_clskey
startprioss        r   from_crawlerz ScrapyPriorityQueue.from_crawler5       s70#zBBBr   c                 v    || _         || _        || _        i | _        d | _        |                     |           d S r   )r    r!   r"   queuescurprio
init_prios)selfr    r!   r"   r#   s        r   __init__zScrapyPriorityQueue.__init__9   s>    $8!
#####r   c                 z    |sd S |D ]}|                      |          | j        |<    t          |          | _        d S r   )qfactoryr'   minr(   )r*   r#   prioritys      r   r)   zScrapyPriorityQueue.init_priosA   sJ     	F" 	< 	<H$(MM($;$;DK!!:r   c           	      j    t          | j        d | j        | j        dz   t	          |          z             S N/)r   r!   r    r"   str)r*   r"   s     r   r-   zScrapyPriorityQueue.qfactoryJ   s6    %LHsNSXX%	
 
 	
r   c                     |j          S r   )r/   r*   requests     r   r/   zScrapyPriorityQueue.priorityR   s       r   c                     |                      |          }|| j        vr|                     |          | j        |<   | j        |         }|                    |           | j        || j        k     r	|| _        d S d S r   )r/   r'   r-   pushr(   )r*   r6   r/   qs       r   r8   zScrapyPriorityQueue.pushU   s    ==))4;& 	<$(MM($;$;DK!K!	w< 	$8dl#: 	$#DLLL	$ 	$r   c                     | j         d S | j        | j                  }|                                }|s\| j        | j         = |                                 d | j                                        D             }|rt          |          nd | _         |S )Nc                     g | ]	\  }}||
S r   r   )r	   pr9   s      r   r   z+ScrapyPriorityQueue.pop.<locals>.<listcomp>f   s!    <<<41a!<Q<<<r   )r(   r'   popcloseitemsr.   )r*   r9   mprioss       r   r=   zScrapyPriorityQueue.pop^   s    < 	FK%EEGG 	9DL)GGIII<<4;#4#4#6#6<<<E).83u:::DDLr   c                 `    | j         dS | j        | j                  }|                                S )  Returns the next object to be returned by :meth:`pop`,
        but without removing it from the queue.

        Raises :exc:`NotImplementedError` if the underlying queue class does
        not implement a ``peek`` method, which is optional for queues.
        N)r(   r'   peek)r*   queues     r   rD   zScrapyPriorityQueue.peekj   s/     < 	4DL)zz||r   c                     g }| j                                         D ].\  }}|                    |           |                                 /|S r   )r'   r?   appendr>   )r*   activer<   r9   s       r   r>   zScrapyPriorityQueue.closev   sN    K%%'' 	 	DAqMM!GGIIIIr   c                 t    | j         r0t          d | j                                         D                       ndS )Nc              3   4   K   | ]}t          |          V  d S r   lenr	   xs     r   	<genexpr>z.ScrapyPriorityQueue.__len__.<locals>.<genexpr>~   s(      88a3q66888888r   r   )r'   sumvaluesr*   s    r   __len__zScrapyPriorityQueue.__len__}   s9    <@KNs884;#5#5#7#7888888QNr   Nr   )__name__
__module____qualname____doc__classmethodr$   r+   r)   r-   r/   r8   r=   rD   r>   rS   r   r   r   r   r      s         2 C C C [C$ $ $ $' ' '
 
 
! ! !$ $ $
 
 

 
 
  O O O O Or   r   c                   &    e Zd Zd Zd Zd Zd ZdS )DownloaderInterfacec                 (    |j         j        | _        d S r   )engine
downloader)r*   r    s     r   r+   zDownloaderInterface.__init__   s    !.3r   c                        fd|D             S )Nc                 >    g | ]}                     |          |fS r   )_active_downloads)r	   slotr*   s     r   r   z-DownloaderInterface.stats.<locals>.<listcomp>   s,    PPP''--t4PPPr   r   )r*   possible_slotss   ` r   statszDownloaderInterface.stats   s    PPPPPPPPr   c                 8    | j                             |d           S r   )r^   _get_slot_keyr5   s     r   get_slot_keyz DownloaderInterface.get_slot_key   s    ,,Wd;;;r   c                 j    || j         j        vrdS t          | j         j        |         j                  S )z<Return a number of requests in a Downloader for a given slotr   )r^   slotsrL   rH   r*   rb   s     r   ra   z%DownloaderInterface._active_downloads   s4    t,, 	14?(.5666r   N)rU   rV   rW   r+   rd   rg   ra   r   r   r   r[   r[      sS        4 4 4Q Q Q< < <7 7 7 7 7r   r[   c                   ^    e Zd ZdZedd            ZddZddZd Zd Z	d Z
d	 Zd
 Zd ZdS )DownloaderAwarePriorityQueuezPriorityQueue which takes Downloader activity into account:
    domains (slots) with the least amount of active downloads are dequeued
    first.
    r   c                      | ||||          S r   r   r   s        r   r$   z)DownloaderAwarePriorityQueue.from_crawler   r%   r   c                    |j                             d          dk    rt          d| j         d          |r-t	          |t
                    st          d|j        d          t          |          | _        || _        || _	        || _
        i | _        |pi                                 D ]#\  }}|                     ||          | j        |<   $d S )NCONCURRENT_REQUESTS_PER_IPr   "z-" does not support CONCURRENT_REQUESTS_PER_IPzDDownloaderAwarePriorityQueue accepts ``slot_startprios`` as a dict; z instance is passed. Most likely, it means the state iscreated by an incompatible priority queue. Only a crawl started with the same priority queue class can be resumed.)settingsgetint
ValueError	__class__
isinstancedictr[   _downloader_interfacer!   r"   r    pqueuesr?   	pqfactory)r*   r    r!   r"   slot_startpriosrb   r#   s          r   r+   z%DownloaderAwarePriorityQueue.__init__   s	   ""#?@@AE 	QDNQQQ    		:ot#D#D 		.",. . .   &9%A%A"$8!!0!6B = = ? ? 	B 	BD*!%j!A!ADL	B 	Br   c                 j    t          | j        | j        | j        dz   t	          |          z   |          S r1   )r   r    r!   r"   r   )r*   rb   r#   s      r   ry   z&DownloaderAwarePriorityQueue.pqfactory   s8    "L%HsNZ---	
 
 	
r   c                     | j                             | j                  }|sd S t          |          d         }| j        |         }|                                }t          |          dk    r| j        |= |S )N   r   )rw   rd   rx   r.   r=   rL   )r*   rd   rb   rE   r6   s        r   r=   z DownloaderAwarePriorityQueue.pop   sp    *00>> 	F5zz!}T"))++u::? 	#T"r   c                     | j                             |          }|| j        vr|                     |          | j        |<   | j        |         }|                    |           d S r   )rw   rg   rx   ry   r8   )r*   r6   rb   rE   s       r   r8   z!DownloaderAwarePriorityQueue.push   sc    )66w??t|# 	6!%!5!5DLT"

7r   c                     | j                             | j                  }|sdS t          |          d         }| j        |         }|                                S )rC   Nr}   )rw   rd   rx   r.   rD   )r*   rd   rb   rE   s       r   rD   z!DownloaderAwarePriorityQueue.peek   sR     *00>> 	45zz!}T"zz||r   c                 ~    d | j                                         D             }| j                                          |S )Nc                 >    i | ]\  }}||                                 S r   )r>   )r	   rb   rE   s      r   
<dictcomp>z6DownloaderAwarePriorityQueue.close.<locals>.<dictcomp>   s&    NNN+$$NNNr   )rx   r?   clear)r*   rH   s     r   r>   z"DownloaderAwarePriorityQueue.close   s<    NN9K9K9M9MNNNr   c                 t    | j         r0t          d | j                                         D                       ndS )Nc              3   4   K   | ]}t          |          V  d S r   rK   rM   s     r   rO   z7DownloaderAwarePriorityQueue.__len__.<locals>.<genexpr>   s(      99a3q66999999r   r   )rx   rP   rQ   rR   s    r   rS   z$DownloaderAwarePriorityQueue.__len__   s9    =A\Ps994<#6#6#8#8999999qPr   c                     || j         v S r   )rx   rj   s     r   __contains__z)DownloaderAwarePriorityQueue.__contains__   s    t|##r   NrT   )rU   rV   rW   rX   rY   r$   r+   ry   r=   r8   rD   r>   rS   r   r   r   r   rl   rl      s         
 C C C [CB B B B4
 
 
 
        
Q Q Q$ $ $ $ $r   rl   )r   loggingscrapy.utils.miscr   	getLoggerrU   loggerr   r   r[   rl   r   r   r   <module>r      s      - - - - - -		8	$	$2 2 2$cO cO cO cO cO cO cO cOL7 7 7 7 7 7 7 7"W$ W$ W$ W$ W$ W$ W$ W$ W$ W$r   