3de9ddlmZddlZddlZddlZddlZddlmZmZddl m Z ddl m Z  ddl m Z n #e$rdZ YnwxYwddlmZddlmZmZdd lmZdd lmZdd lmZdd lmZdd lmZmZddlm Z ddl!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'ddl(m)Z)m*Z*ddl+m,Z,m-Z-ddl.m/Z/m0Z0m1Z1m2Z2erddl3m4Z4ej5e6Z7GddZ8GddZ9Gdde9Z:dS)) annotationsN) TYPE_CHECKINGOptional)defer)DoesNotImplement)MultipleInvalid) verifyClass)SpidersignalsExecutionEngine)ScrapyDeprecationWarning)ExtensionManager) ISpiderLoader)Settingsoverridden_settings) SignalManager)LogCounterHandlerconfigure_loggingget_scrapy_root_handlerinstall_scrapy_root_handlerlog_reactor_infolog_scrapy_info)create_instance load_object)install_shutdown_handlers signal_names)install_reactoris_asyncio_reactor_installed#verify_installed_asyncio_event_loopverify_installed_reactor)RequestFingerprintercdeZdZd d dZejdZdZdZejd Z dS) CrawlerNF init_reactorboolc t|trtdt|ts|t |}||_||_|j|jt||_ t|jd||_ t||jd tj tt%|j}t&ddt+j|it/t1|j fd|_|j |jtjt|jd}|||_t=t|jd |j| |_|jd }|jd }|r'|rtA||nd dl!m"}tG|r.tI|tKr|rtM|tOj||_(|j)d|_*d|_+d|_,dS)Nz5The spidercls argument must be a class, not an object STATS_CLASS LOG_LEVEL)levelz!Overridden settings: %(settings)ssettingscBtjSN)loggingroot removeHandler)handlers.lib/python3.11/site-packages/scrapy/crawler.pyz"Crawler.__init__..Ps (B(B7(K(K LOG_FORMATTERREQUEST_FINGERPRINTER_CLASS)r+crawlerTWISTED_REACTORASYNCIO_EVENT_LOOPrreactorF)- isinstancer ValueErrordictr spiderclscopyr+update_settingsrr rstatsrgetr.r/ addHandlerrloggerinfopprintpformatrr_Crawler__remove_handlerconnectengine_stopped from_crawler logformatterrrequest_fingerprinterrtwisted.internetr;rr!rr r extensionsfreezecrawlingspiderengine) selfr?r+r%dlf_cls reactor_class event_loopr;r1s @r2__init__zCrawler.__init__4s i ( ( VTUU U h % % * *))H"   &&t}555$T** >[}!=>>tDD #D 0A0A+0N0NOOO ((( $T]33 4 4 0:v~a?P?P2Q    # $ $ 7 ' 6 6 6!L K K K T2G4JKKKT]?;<<"//55;J  &CD E E]< < < "  &78 ]#78   5 z::::444444      @ $] 3 3 3+-- @* @3J???*7==   15 r4c/K|jrtdd|_ |j|i||_||_t |j}|j|j|Vtj |jj VdS#t$r+d|_|j|j VwxYw)NzCrawling already taking placeTF)rR RuntimeError_create_spiderrS_create_enginerTiterstart_requests open_spiderr maybeDeferredstart Exceptionclose)rUargskwargsr`s r2crawlz Crawler.crawlrs = @>?? ?  -$-t>v>>DK--//DK!$+"<"<">">??N+))$+~FF F F F%dk&788 8 8 8 8 8   !DM{ *k'')))))   s BB665C+c.|jj|g|Ri|Sr-)r?rL)rUrfrgs r2r]zCrawler._create_spiders'*t~*4A$AAA&AAAr4c*tfdS)Nc,Sr-stop)_rUs r2r3z(Crawler._create_engine..styy{{r4r rUs`r2r^zCrawler._create_enginest%:%:%:%:;;;r4c#jK|jr)d|_tj|jjVdSdS)zoStarts a graceful stop of the crawler and returns a deferred that is fired when the crawler is stopped.FN)rRrrbrTrmros r2rmz Crawler.stopsE = 8!DM%dk&677 7 7 7 7 7 8 8r4NF)r%r&) __name__ __module__ __qualname__rZrinlineCallbacksrhr]r^rmr4r2r$r$3s<6<6<6<6<6| "BBB<<< 88888r4r$ceZdZdZeddZedZddZedZ d Z d Z d Z d Z d ZejdZdS) CrawlerRunnera This is a convenient helper class that keeps track of, manages and runs crawlers inside an already setup :mod:`~twisted.internet.reactor`. The CrawlerRunner object must be instantiated with a :class:`~scrapy.settings.Settings` object. This class shouldn't be needed (since Scrapy is responsible of using it accordingly) unless writing scripts that manually handle the crawling process. See :ref:`run-from-script` for an example. c|jSr-) _crawlersros r2r3zCrawlerRunner.sT^r4zeSet of :class:`crawlers ` started by :meth:`crawl` and managed by this class.)doccL|d}t|}trttfnt} t t |n'#|$rt jdtdYnwxYw| | S)z'Get SpiderLoader instance from settingsSPIDER_LOADER_CLASSzSPIDER_LOADER_CLASS (previously named SPIDER_MANAGER_CLASS) does not fully implement scrapy.interfaces.ISpiderLoader interface. Please add all missing methods to avoid unexpected runtime errors.category stacklevel) rCrrrr rwarningswarnr from_settings frozencopy)r+cls_path loader_clsexcss r2_get_spider_loaderz CrawlerRunner._get_spider_loaders<< 566 ** 3B X  / /HX    z 2 2 2 2    MU2        ''(;(;(=(=>>>sA!A<;A<Nct|ts|t|}||_|||_t |_t |_d|_ dSrq) r<r>rr+r spider_loadersetrz_activebootstrap_failed)rUr+s r2rZzCrawlerRunner.__init__si h % % * *))H  !44X>>uu %r4cHtjdtd|jS)NzJCrawlerRunner.spiders attribute is renamed to CrawlerRunner.spider_loader.r~r)rrrrros r2spiderszCrawlerRunner.spiderss1  +-     !!r4ct|trtd||}|j|g|Ri|S)ae Run a crawler with the provided arguments. It will call the given Crawler's :meth:`~Crawler.crawl` method, while keeping track of it so it can be stopped later. If ``crawler_or_spidercls`` isn't a :class:`~scrapy.crawler.Crawler` instance, this method will try to create one using this parameter as the spider class given to it. Returns a deferred that is fired when the crawling is finished. :param crawler_or_spidercls: already created crawler, or a spider class or spider's name inside the project to create it :type crawler_or_spidercls: :class:`~scrapy.crawler.Crawler` instance, :class:`~scrapy.spiders.Spider` subclass or string :param args: arguments to initialize the spider :param kwargs: keyword arguments to initialize the spider lThe crawler_or_spidercls argument cannot be a spider object, it must be a spider class (or a Crawler object))r<r r=create_crawler_crawl)rUcrawler_or_spiderclsrfrgr7s r2rhzCrawlerRunner.crawlsg, *F 3 3 B %%&:;;t{74T444V444r4cjj|i|jfd}|S)Ncjjxjt dd zc_|S)NrS)crawlersdiscardrrgetattr)resultr7rVrUs r2_donez#CrawlerRunner._crawl.._donesW M ! !' * * * L  # # #  ! !(D)I)I%I I ! !Mr4)raddrhraddBoth)rUr7rfrgrrVs`` @r2rzCrawlerRunner._crawls~ '""" GM4 *6 * *         yyr4ct|trtdt|tr|S||S)a Return a :class:`~scrapy.crawler.Crawler` object. * If ``crawler_or_spidercls`` is a Crawler, it is returned as-is. * If ``crawler_or_spidercls`` is a Spider subclass, a new Crawler is constructed for it. * If ``crawler_or_spidercls`` is a string, this function finds a spider with this name in a Scrapy project (using spider loader), then creates a Crawler instance for it. r)r<r r=r$_create_crawler)rUrs r2rzCrawlerRunner.create_crawlers_ *F 3 3 B  *G 4 4 (' '##$8999r4ct|tr|j|}t ||jSr-)r<strrloadr$r+)rUr?s r2rzCrawlerRunner._create_crawler s< i % % ;*// ::Iy$-000r4cbtjdt|jDS)z Stops simultaneously all the crawling jobs taking place. Returns a deferred that is fired when they all have ended. c6g|]}|Srvrl).0cs r2 z&CrawlerRunner.stop..s "I"I"I16688"I"I"Ir4)r DeferredListlistrros r2rmzCrawlerRunner.stops. !"I"IT$-5H5H"I"I"IJJJr4c#`K|jr$tj|jV|j"dSdS)z join() Returns a deferred that is fired when all managed :attr:`crawlers` have completed their executions. N)rrrros r2joinzCrawlerRunner.joinsKl 3$T\22 2 2 2l 3 3 3 3 3r4r-)rrrsrt__doc__propertyr staticmethodrrZrrhrrrrmrrurrvr4r2rxrxs  x## 3H ??\?&&&&&""X"555<    :::(111 KKK 33333r4rxcLeZdZdZd fd ZdZdZdZd dZd Z d d Z xZ S)CrawlerProcessa A class to run multiple scrapy crawlers in a process simultaneously. This class extends :class:`~scrapy.crawler.CrawlerRunner` by adding support for starting a :mod:`~twisted.internet.reactor` and handling shutdown signals, like the keyboard interrupt command Ctrl-C. It also configures top-level logging. This utility should be a better fit than :class:`~scrapy.crawler.CrawlerRunner` if you aren't running another :mod:`~twisted.internet.reactor` within your application. The CrawlerProcess object must be instantiated with a :class:`~scrapy.settings.Settings` object. :param install_root_handler: whether to install root logging handler (default: True) This class shouldn't be needed (since Scrapy is responsible of using it accordingly) unless writing scripts that manually handle the crawling process. See :ref:`run-from-script` for an example. NTct|t|j|t |jd|_dSrq)superrZrr+r_initialized_reactor)rUr+install_root_handler __class__s r2rZzCrawlerProcess.__init__=sL """$-)=>>> &&&$)!!!r4cddlm}t|jt|}t dd|i||jdS)Nrr:zDReceived %(signame)s, shutting down gracefully. Send again to force signame) rOr;r _signal_killrrErFcallFromThread_graceful_stop_reactorrUsignumrnr;rs r2_signal_shutdownzCrawlerProcess._signal_shutdownCso,,,,,,!$"3444v& R      t:;;;;;r4cddlm}ttjt |}t dd|i||j dS)Nrr:z4Received %(signame)s twice, forcing unclean shutdownr) rOr;rsignalSIG_IGNrrErFr _stop_reactorrs r2rzCrawlerProcess._signal_killNsn,,,,,,!&.111v& BYPWDX    t122222r4ct|tr|j|}|j }d|_t ||j|S)NT)r%)r<rrrrr$r+)rUr?r%s r2rzCrawlerProcess._create_crawlerXsT i % % ;*// ::I44 $(!y$-lKKKKr4c$ddlm}|r7|}|jrdS||j|rt |jt|j d}t||j ||}| | }| |j d|dd|j|d dS) aV This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache based on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`. If ``stop_after_crawl`` is True, the reactor will be stopped after all crawlers have finished, using :meth:`join`. :param bool stop_after_crawl: stop or not the reactor when all crawlers have finished :param bool install_signal_handlers: whether to install the shutdown handlers (default: True) rr:N DNS_RESOLVERREACTOR_THREADPOOL_MAXSIZE) maxthreadsbeforeshutdownF)installSignalHandlers)rOr;rcalledrrrrrr+rinstall_on_reactor getThreadPooladjustPoolsizegetintaddSystemEventTriggerrmrun)rUstop_after_crawlinstall_signal_handlersr;rVresolver_classresolvertps r2rczCrawlerProcess.start_s -,,,,,  * Ax  IId( ) ) ) " = %d&; < < <$T]>%BCC">4=$PWXXX##%%%  " " $ $ T]%9%9:V%W%WXXX%%h DIFFF % 00000r4cb|}||j|Sr-)rmrr)rUrVs r2rz%CrawlerProcess._graceful_stop_reactors) IIKK $$%%%r4c^ddlm} |dS#t$rYdSwxYw)Nrr:)rOr;rmr\)rUrnr;s r2rzCrawlerProcess._stop_reactorsL,,,,,,  LLNNNNN    DD s  ,,)NT)TTr-) rrrsrtrrZrrrrcrr __classcell__)rs@r2rr%s.****** < < <333LLL 1 1 1 1D r4r); __future__rr.rGrrtypingrrrOrzope.interface.exceptionsrr ImportErrorzope.interface.verifyr scrapyr r scrapy.core.enginer scrapy.exceptionsrscrapy.extensionrscrapy.interfacesrscrapy.settingsrrscrapy.signalmanagerrscrapy.utils.logrrrrrrscrapy.utils.miscrrscrapy.utils.ossignalrrscrapy.utils.reactorrrr r!scrapy.utils.requestr" getLoggerrrrEr$rxrrvr4r2rs"""""" ********""""""6666669999999OOO.-----""""""""......666666------++++++99999999......;:::::::IIIIIIII:999999  8 $ $]8]8]8]8]8]8]8]8@O3O3O3O3O3O3O3O3dggggg]gggggs 3==