3dkdZddlZddlZddlmZddlmZmZmZddl m Z ddl m Z ddl mZddlmZmZGd d ZGd d eZGd dejZdS)z Base class for Scrapy commands N)Path)AnyDictOptional)failure)CrawlerProcess) UsageError)arglist_to_dictfeed_process_params_from_cliceZdZUdZdZeeed<iZe e e fed<dZ ddZ dZd Zd Zd Zd Zd ZdZdZdS) ScrapyCommandFNcrawler_processdefault_settingsrreturncd|_dSN)settingsselfs 8lib/python3.11/site-packages/scrapy/commands/__init__.py__init__zScrapyCommand.__init__s ! cRt|drtd||_dS)N_crawlerzcrawler already set)hasattr RuntimeErrorr)rcrawlers r set_crawlerzScrapyCommand.set_crawlers. 4 $ $ 6455 5 rcdS)zT Command syntax (preferably one-line). Do not include command name. rs rsyntaxzScrapyCommand.syntax" rrcdS)z4 A short description of the command r r!rs r short_desczScrapyCommand.short_desc(r#rc*|S)zA long description of the command. Return short description when not available. It cannot contain newlines since contents will be formatted by optparser which removes newlines and wraps text. )r%rs r long_desczScrapyCommand.long_desc.s    rc*|S)zAn extensive help for the command. It will be shown when using the "help" command. It can contain newlines since no post-formatting will be applied to its contents. )r'rs rhelpzScrapyCommand.help5s ~~rc |d}|ddd|ddd d d |jd d |ddd|ddd d|ddd|dddgdd|dddd S)zO Populate option parse with options available for this command Global Options)titlez --logfileFILEz(log file. if omitted stderr will be usedmetavarr)z-Lz --loglevelLEVELNzlog level (default: LOG_LEVEL))r/defaultr)z--nolog store_truezdisable logging completely)actionr)z --profilez#write python cProfile stats to FILEz --pidfilezwrite process ID to FILEz-sz--setappend NAME=VALUEz&set/override setting (may be repeated))r5r3r/r)z--pdbzenable pdb on failure)add_argument_group add_argumentr)rparsergroups r add_optionszScrapyCommand.add_options<sC))0@)AA  .X       E k(BEEE      l1M      6    ;=WXXX    9     7<>UVVVVVrc |jt|jdn#t$rt ddwxYw|jr?|jddd|jd|jd|jr?|jddd|jd |jd|jr|jddd|j rTt|j  ttjtjzd |jrt#jdSdS) Ncmdlinepriorityz#Invalid -s value, use -s NAME=VALUEF print_help LOG_ENABLEDTLOG_FILEr1zutf-8)encoding)rsetdictr set ValueErrorr logfileloglevelnologpidfiler write_textstrosgetpidlineseppdbrstartDebugModerargsoptss rprocess_optionszScrapyCommand.process_options_s V M ! !/$(";";i ! P P P P V V VBuUUU U V < L M  mTI  F F F M  j$,  K K K = N M  mTI  F F F M  k4=9  M M M : H M  mUY  G G G <     ) )BIKK  2:- *    8 %  " $ $ $ $ $ % %s .1A ct)z2 Entry point for running commands )NotImplementedErrorrTs rrunzScrapyCommand.runxs "!r)rN)__name__ __module__ __qualname__requires_projectrrr__annotations__rrrNrexitcoderrr"r%r'r)r<rWrZr!rrr r s04OXn-444(*d38n)))H""""     !!!   !W!W!WF%%%2"""""rr ceZdZdZdZdZdS)BaseRunSpiderCommandzb Common class used to share functionality between the crawl, parse and runspider commands c t|||dddgdd|ddd dd |d d d dd |dddddS)Nz-aspargsr6r7z%set spider argument (may be repeated))destr5r3r/r)z-oz--outputr-zappend scraped items to the end of FILE (use - for stdout), to define format set a colon at the end of the output URI (i.e. -o FILE:FORMAT))r/r5r)z-Oz--overwrite-outputzdump scraped items into FILE, overwriting any existing file, to define format set a colon at the end of the output URI (i.e. -O FILE:FORMAT)z-tz--output-formatFORMATzformat to use for dumping itemsr.)r r<r9)rr:s rr<z BaseRunSpiderCommand.add_optionss!!$///  8       _       _       2      rcXt||| t|j|_n#t$rt ddwxYw|js|jrEt|j |j|j |j}|j d|ddSdS)Nz#Invalid -a value, use -a NAME=VALUEFrAFEEDSr>r?) r rWr rdrHr outputoverwrite_outputr r output_formatrG)rrUrVfeedss rrWz$BaseRunSpiderCommand.process_optionss%%dD$777 V)$+66DKK V V VBuUUU U V ; B$/ B0  "% E M  guy  A A A A A B Bs 8AN)r[r\r]__doc__r<rWr!rrrbrbsB   B B B B B Brrbc4eZdZdZdfd ZfdZdZxZS) ScrapyHelpFormatterz? Help Formatter for scrapy command line help messages. NcRt||||dS)N)indent_incrementmax_help_positionwidth)superr)rprogrsrtru __class__s rrzScrapyHelpFormatter.__init__s:  -/      rcp||}t|Sr)format_part_stringsrv _join_parts)r part_stringspartsrxs rr{zScrapyHelpFormatter._join_partss.((66ww""5)))rc r>ddr#ddtddzd<fdttD}|dddD]|}d|vrdnd }|dd |<d d |t|zd g}|d z|}S)zM Underline and title case command line help message headers. rzusage: zUsage ===== NcJg|]}|d| S)z: )endswith).0ir|s r z;ScrapyHelpFormatter.format_part_strings..s@   <?3K3KE3R3R    rr+-=r  ) startswithlenranger,joininsert)rr|headingsindexchar underlines ` rrzz'ScrapyHelpFormatter.format_part_stringss'  ULO66yAA U0<?3y>>CSCS3TTLO    S..//   ddd^ 6 6E*l5.AAJ33sD".u"5crc":"@"@"B"BL s<3F/G/G(G$ OPPI    9 5 5 5 5r)rprqN)r[r\r]rmrr{rz __classcell__)rxs@rroroso      *****rro)rmargparserOpathlibrtypingrrrtwisted.pythonrscrapy.crawlerrscrapy.exceptionsr scrapy.utils.confr r r rb HelpFormatterror!rrrs< &&&&&&&&&&""""""))))))((((((KKKKKKKKl"l"l"l"l"l"l"l"^3B3B3B3B3B=3B3B3Bl(0r