
    3 d"                         d dl Z d dlZd dlZd dlmZ d dlmZ d dlmZm	Z	 d dl
mZ d dlZd dlmZ d dlmZ d dlmZmZ d	 Zd
 Z G d de          ZdS )    N)import_module)Path)Optionalcast)urlparse)ScrapyCommand)
UsageError)render_templatefilestring_camelcasec                     |                      dd                               dd          } | d         t          j        vrd| z   } | S )zSanitize the given module name, by replacing dashes and points
    with underscores and prefixing it with a letter if it doesn't start
    with one
    -_.r   a)replacestringascii_letters)module_names    9lib/python3.11/site-packages/scrapy/commands/genspider.pysanitize_module_namer      sM    
 %%c3//77SAAK1~V11 (K'    c                     t          |           }|j        dk    r0|j        dk    r%t          d|                     d          z             }|j        S )z#Extract domain name from URL string z///)r   schemenetloclstrip)urlos     r   extract_domainr       sK    Ax2~ -!(b. -TCJJsOO+,,8Or   c                       e Zd ZdZddiZd Zd Zd Zd Zd Z	de
d	ee         fd
Zd Zde
d	efdZed	e
fd            ZdS )CommandFLOG_ENABLEDc                     dS )Nz[options] <name> <domain> selfs    r   syntaxzCommand.syntax'   s    **r   c                     dS )Nz/Generate new spider using pre-defined templatesr%   r&   s    r   
short_desczCommand.short_desc*   s    @@r   c                 2   t          j        | |           |                    ddddd           |                    ddd	dd
           |                    ddddd           |                    ddddd           |                    dddd           d S )Nz-lz--listlist
store_truezList available templates)destactionhelpz-ez--editeditzEdit spider after creating itz-dz--dumpdumpTEMPLATEz Dump template to standard output)r.   metavarr0   z-tz
--templatetemplatebasiczUses a custom template.)r.   defaultr0   z--forceforcez<If the spider already exists, overwrite it with the template)r   add_optionsadd_argument)r'   parsers     r   r9   zCommand.add_options-   s    !$///+ 	 	
 	
 	
 	0 	 	
 	
 	
 	3 	 	
 	
 	
 	* 	 	
 	
 	
 	O	 	 	
 	
 	
 	
 	
r   c                    |j         r|                                  d S |j        rA|                     |j                  }|r#t	          |                    d                     d S t          |          dk    rt                      |dd         \  }}t          |          }t          |          }| j
                            d          |k    rt	          d           d S |j        s|                     |          rd S |                     |j                  }|rD|                     ||||j        |           |j        r!t#          j        d| d          | _        d S d S d S )	Nzutf-8)encoding   r   BOT_NAMEz9Cannot create a spider with the same name as your projectzscrapy edit "")r,   _list_templatesr2   _find_templateprint	read_textlenr	   r    r   settingsgetr8   _spider_existsr5   
_genspiderr1   ossystemexitcode)r'   argsoptstemplate_filenamer   domainmodules           r   runzCommand.runR   s   9 	  """F9 	 //	::M Am--w-??@@@Ft99> 	,,1I	c$$%d++=Z((F2 	MNNNFz 	d11$77 	F++DM:: 	COOFD&$-OOOy C "	*A$*A*A*A B B	C 	CC Cr   c                    d                     d |                    d          D                       }| j                            d          t	          | j                            d                    |||| dd}| j                            d          rFt          | j        d                   }t          |j                  j        	                                }	nd}t          d	          }	|	|z   d
}
t          j        ||
           t          |
fi | t          d|d|d|rdnd           |rt          d|j         d	|            dS dS )z7Generate the spider module, based on the given templater   c              3   >   K   | ]}|                                 V  d S )N)
capitalize).0ss     r   	<genexpr>z%Command._genspider.<locals>.<genexpr>q   s*      $O$OQ\\^^$O$O$O$O$O$Or   r   r?   Spider)project_nameProjectNamerR   rP   rQ   	classnameNEWSPIDER_MODULENr   .pyzCreated spider z using template  
)endzin module:
  )joinsplitrF   rG   r   r   r   __file__parentresolveshutilcopyfiler
   rC   __name__)r'   rR   rP   rQ   template_namerO   capitalized_moduletvarsspiders_modulespiders_dirspider_files              r   rI   zCommand._genspidero   s   WW$O$OV\\#=N=N$O$O$OOO M--j99+DM,=,=j,I,IJJ.666
 
 =/00 	$*4=9K+LMMN~677>FFHHKK!Ns))K$v-222{333K115111HdHHmHHH%/4	
 	
 	
 	
  	GE>#:EEVEEFFFFF	G 	Gr   r5   returnc                     t          | j        | d          }|                                r|S t          d| d           t          d           d S )N.tmplzUnable to find template: ra   z=Use "scrapy genspider --list" to see all available templates.)r   templates_direxistsrC   )r'   r5   rO   s      r   rB   zCommand._find_template   se    T/H1C1C1CDD!! 	!  6(666777MNNNtr   c                     t          d           t          t          | j                                                            D ]$}|j        dk    rt          d|j                    %d S )NzAvailable templates:rs     )rC   sortedr   rt   iterdirsuffixstem)r'   files     r   rA   zCommand._list_templates   sr    $%%%4 233;;==>> 	( 	(D{g% (&49&&'''	( 	(r   rP   c                    | j                             d          sNt          |dz             }|                                r&t	          |                                 d           dS dS | j        
J d            	 | j        j                            |          }t	          d|d           t	          d	|j	                    dS # t          $ r Y nw xY wt          | j         d                   }t          t          t          |j                            j        }|                                }||dz   z  }|                                rt	          | d           dS dS )
Nr^   r_   z already existsTFz.crawler_process must be set before calling runzSpider z already exists in module:rw   )rF   rG   r   ru   rC   rg   crawler_processspider_loaderload
__module__KeyErrorr   r   strre   rf   )r'   rP   path	spiderclsrn   ro   spiders_dir_abss          r   rH   zCommand._spider_exists   s   }  !344 	u%%D{{}} 888999t5  	< 	<;	< 	< 	<	,:??EEI
 >D>>>???-y+--...4  	 	 	D	 't}5G'HII4^%<==>>E%--//$,/;;== 	T***+++4us   ;C 
CCc                     t          t          | j        d         pt          t          j        d         d          d                    S )NTEMPLATES_DIRr   	templatesspiders)r   r   rF   scrapy__path__r&   s    r   rt   zCommand.templates_dir   sD    o.W$vq7I;2W2W 
 
 	
r   N)rj   r   __qualname__requires_projectdefault_settingsr(   r*   r9   rS   rI   r   r   r   rB   rA   boolrH   propertyrt   r%   r   r   r"   r"   "   s        %u-+ + +A A A#
 #
 #
JC C C:G G G6s x~    ( ( ( 3  4        D 
s 
 
 
 X
 
 
r   r"   )rJ   rh   r   	importlibr   pathlibr   typingr   r   urllib.parser   r   scrapy.commandsr   scrapy.exceptionsr	   scrapy.utils.templater
   r   r   r    r"   r%   r   r   <module>r      s   				   # # # # # #       ! ! ! ! ! ! ! ! ! ! ! ! ! !  ) ) ) ) ) ) ( ( ( ( ( ( G G G G G G G G    _
 _
 _
 _
 _
m _
 _
 _
 _
 _
r   