3d ddlZddlZddlmZddlmZddlmZddlm Z ddl m Z eeGddZ dS) N) defaultdict) implementer) ISpiderLoader) walk_modules)iter_spider_classescReZdZdZdZdZdZdZedZ dZ dZ d Z d S) SpiderLoaderzZ SpiderLoader is a class which locates and loads spiders in a Scrapy project. c|d|_|d|_i|_t t |_|dS)NSPIDER_MODULESSPIDER_LOADER_WARN_ONLY) getlistspider_modulesgetbool warn_only_spidersrlist_found_load_all_spiders)selfsettingss 3lib/python3.11/site-packages/scrapy/spiderloader.py__init__zSpiderLoader.__init__s]&../?@@!))*CDD !$''       cg}|jD]'\|fdD(|r6d|}t jd|dt dSdS)Nc Tg|]$\}}tdkd|dd|d%S)z z named z (in ))len).0modcls locationsnames r z7SpiderLoader._check_name_duplicates..sY S9~~)888T88#888rz z/There are several spiders with the same name: z' This can cause unexpected behavior.category)ritemsextendjoinwarningswarn UserWarning)rdupes dupes_stringr"r#s @@r_check_name_duplicatesz#SpiderLoader._check_name_duplicatess#{0022  OD) LL$-      !;;u--L MKKKK$        rct|D]B}|j|j|j|jf||j|j<CdSN)rrr#append__name__r)rmodulespclss r _load_spiderszSpiderLoader._load_spiders-sZ(00 . .E K # * *FOU^+L M M M(-DM%* % % . .rc *|jD]v} t|D]}||+#t$r?|jr4t jdtjd|dtnYswxYw| dS)N z$Could not load spiders from module 'z#'. See above traceback for details.r%) rrr6 ImportErrorrr*r+ traceback format_excRuntimeWarningr/)rr#r4s rrzSpiderLoader._load_all_spiders2s'  D *400//F&&v..../   >M;Y133;;(,;;;"0   ##%%%%%s'3AA<;A<c||Sr1)r!rs r from_settingszSpiderLoader.from_settingsCss8}}rc^ |j|S#t$rtd|wxYw)z Return the Spider class for the given spider name. If the spider name is not found, raise a KeyError. zSpider not found: )rKeyError)r spider_names rloadzSpiderLoader.loadGsF  ?=- - ? ? ?= ==>> > ?s ,cNfd|jDS)zT Return the list of spider names that can handle the given request. cDg|]\}}||Sr>)handles_request)rr#r!requests rr$z0SpiderLoader.find_by_request..Us@   T33;N;Nw;W;W    r)rr')rrGs `rfind_by_requestzSpiderLoader.find_by_requestQs<    "&-"5"5"7"7    rcNt|jS)zW Return a list with the names of all spiders available in the project. )rrkeys)rs rrzSpiderLoader.listYs DM&&(()))rN) r3 __module__ __qualname____doc__rr/r6r classmethodr?rCrHrr>rrr r s !!!&... &&&"[???   *****rr ) r:r* collectionsrzope.interfacerscrapy.interfacesrscrapy.utils.miscrscrapy.utils.spiderrr r>rrrTs######&&&&&&++++++******333333 ]P*P*P*P*P*P*P*P*P*P*r