
    +d~                        d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZmZ d dl	m
Z
 ddlmZ ddlmZmZ ddlmZ ddlmZmZmZmZ d	d
lmZ d	dlmZmZ d	dlmZ d	dlmZmZmZm Z m!Z!  ej"        d          Z# G d de          Z$ G d de          Z% G d de&          Z'd Z(d Z) G d de          Z* G d de          Z+ G d de          Z, G d de          Z- G d d e          Z. ed!e*d"#            ed$e+d"#           dS )%    N)get_filesystem_class
open_files)split_protocol   )__version__)get_plugin_classregister_driver)load_plugins_from_module)DictSerialiseMixin	classnamemake_path_posix	yaml_load   )
exceptions)Catalog
DataSource)CatalogEntry)COERCION_RULES_has_catalog_dircoerceexpand_defaults
merge_parsintakec                   8    e Zd ZdZd
dZd ZeZd ZddZd	 Z	dS )UserParametera  
    A user-settable item that is passed to a DataSource upon instantiation.

    For string parameters, default may include special functions ``func(args)``,
    which *may* be expanded from environment variables or by executing a shell
    command.

    Parameters
    ----------
    name: str
        the key that appears in the DataSource argument strings
    description: str
        narrative text
    type: str
        one of list ``(COERSION_RULES)``
    default: type value
        same type as ``type``. It a str, may include special functions
        env, shell, client_env, client_shell.
    min, max: type value
        for validation of user input
    allowed: list of type
        for validation of user input
    Nc                     | _         | _        |pt          d         |          j         _        | _        | _        | _        | _        	 t           j        |           _
        n# t          t          f$ r
 d  _
        Y nw xY w j
         _         j        rt           j         j                   _         j        rt           j         j                   _         j        r |dk    r fd j        D              _        d S d S d S )Ntypemlistc                 :    g | ]}t          j        |          S  )r   r   ).0itemselfs     4lib/python3.11/site-packages/intake/catalog/local.py
<listcomp>z*UserParameter.__init__.<locals>.<listcomp>L   s%    MMMF49d33MMM    )namedescription__builtins____name__r   minmaxallowed_defaultr   default
ValueError	TypeErrorexpanded_default)r#   r'   r(   r   r/   r+   r,   r-   s   `       r$   __init__zUserParameter.__init__6   s'   	&BL099B		 !$)W55DLLI& 	  	  	 DLLL	  $8 	3di22DH8 	3di22DH< 	NDGO 	NMMMMMMMDLLL	N 	N 	N 	Ns   A) )BBc                 2    d| j         j         d| j        dS )N< >)	__class__r*   r'   r#   s    r$   __repr__zUserParameter.__repr__N   s"    ;4>*;;TY;;;;r&   c                 l    | j         | j        | j        pdd}dD ]}t          | |          }||||<   |S )z Information about this parameterunknown)r'   r(   r   )r+   r,   r-   r/   )r'   r(   r   getattr)r#   descattrvs       r$   describezUserParameter.describeS   s\     I+I*	
 
 9 	 	Dd##A T
r&   FTc           	          t          | j        t                    s| j        | _        dS t	          | j        t          | j        |||                    | _        dS )z8Compile env, client_env, shell and client_shell commandsN)
isinstancer.   strr2   r   r   r   )r#   clientgetenvgetshells       r$   r   zUserParameter.expand_defaultsa   sT    $--- 	p$(MD!!!$*49odmU[]cem6n6n$o$oD!!!r&   c                    | j         t          | j         |          }| j         dk    r |D ]}|| j        vrt          d|          |S | j        ,|| j        k     r!t          | j        d|d| j                  | j        ,|| j        k    r!t          | j        d|d| j                  | j        P|| j        vrGt          | j        d|dd                    t          t          | j                                      |S )	z'Does value meet parameter requirements?Nr   zItem %s not in allowed list=z is less than z is greater than z# is not one of the allowed values: ,)
r   r   r-   r0   r+   r'   r,   joinmaprD   )r#   valuer@   s      r$   validatezUserParameter.validateh   sR   9 	-49e,,E9 	 G GDL( G$%BAFFFGL8 	UEDH$4 	U			555$((STTT8 	XEDH$4 	X4999eeeTXXVWWW< 	BT\(A 	BTYYYX]X]X]_b_g_ghkloquq}h~h~___  A  B  B  Br&   )NNNNNN)FTT)
r*   
__module____qualname____doc__r3   r:   __str__rA   r   rN   r    r&   r$   r   r      s         0N N N N0< < < G  p p p p    r&   r   c            	       d     e Zd ZdZdi g g i ddddf	 fd	Zed             Zd Zd Zd	 Z	d
 Z
 xZS )LocalCatalogEntryz#A catalog entry on the local systemT Nc                    || _         d| _        || _        | _        || _        || _        || _        || _        |pi | _        |	| _	        d| _
        || _        t          t                    r;t                    }||gng | _        t!          d | j        D                       }n(t          t"                    rFd D             | _        d | j        D             | _        t!          d | j        D                       }nt          t$                    rlfdD             | _        d | j                                        D             | _        t!          d | j                                        D                       }nLt+          j                  r&t/          t0                    rg| _        j        h}nt5          d	z            t7          |          d
k    rt9          d          t7          |          dk    r%t:                              d| j        z             dg}t#          |          d         | _         tC          tD          |           #                    |
|           dS )av  

        Parameters
        ----------
        name: str
            How this entry is known, normally from its key in a YAML file, or
            if that is not provided then from name of file, or name of dir if
            file name is 'catalog.yaml' or 'catalog.yml'.
        description: str
            Brief text about the target source
        driver: str, list, dict or DataSource subclass
            The plugin(s) that can load this. Can be a simple name like "csv",
            which will be looked up in the registry, a fully-qualified class
            name ("package.mod.Class"), a list of these which would all work,
            a dictionary of the same with reasonable names, or an explicit
            class derived from DataSource.
        direct_access: bool
            Is the client allowed to attempt to reach this data
        args: dict
            Passed when instantiating the plugin DataSource
        parameters: list
            UserParameters that can be set
        metadata: dict
            Additional information about this data
        catalog_dir: str
            Location of the catalog, if known
        getenv: bool
            Can parameter default fields take values from the environment
        getshell: bool
            Can parameter default fields run shell commands
        catalog: bool
            Catalog object in which this entry belongs
        Nc              3   $   K   | ]}|j         V  d S N	containerr!   ps     r$   	<genexpr>z-LocalCatalogEntry.__init__.<locals>.<genexpr>   $      ??QQ[??????r&   c                 ,    g | ]}t          |          S r    r   )r!   ds     r$   r%   z.LocalCatalogEntry.__init__.<locals>.<listcomp>   s!    @@@A,Q//@@@r&   c                     g | ]}||S rX   r    r[   s     r$   r%   z.LocalCatalogEntry.__init__.<locals>.<listcomp>   s    EEE!qEAEEEr&   c              3   $   K   | ]}|j         V  d S rX   rY   r[   s     r$   r]   z-LocalCatalogEntry.__init__.<locals>.<genexpr>   r^   r&   c                 H    i | ]}|t          |         d                    S )classr`   )r!   ra   drivers     r$   
<dictcomp>z.LocalCatalogEntry.__init__.<locals>.<dictcomp>   s-    TTTA/q	'0BCCTTTr&   c                     i | ]
\  }}|||S rX   r    r!   kr@   s      r$   rg   z.LocalCatalogEntry.__init__.<locals>.<dictcomp>   s#    SSSTQQSAqSSSr&   c              3   $   K   | ]}|j         V  d S rX   rY   r[   s     r$   r]   z-LocalCatalogEntry.__init__.<locals>.<genexpr>   s$      HHQQ[HHHHHHr&   z5Driver was not a string, list, dict or DataSource: %sr   z7Plugins for a data source must have only one container.r   zNo plugins for entry: %srF   rG   )$_name_default_source_description_driver_direct_access
_open_args_cache_user_parameters	_metadata_catalog_dir_filesystem_catalogrC   rD   r   _pluginsetlistdictitemsvaluesinspectisclass
issubclassr   rZ   r1   lenr0   loggerdebugr'   
_containersuperrT   r3   )r#   r'   r(   rf   direct_accessargscache
parametersmetadatacatalog_dirrF   rG   catalogdr
containersr8   s      `           r$   r3   zLocalCatalogEntry.__init__   s|   H 
#'+ *!R'fc"" 	a!&))B#%9B44rDL??$,?????JJ%% 	a@@@@@DLEEt|EEEDL??$,?????JJ%% 	aTTTTVTTTDLSST\-?-?-A-ASSSDLHH$,2E2E2G2GHHHHHJJ_V$$ 	aFJ)G)G 	a"8DL *+JJVY__```z??Q 	[ YZZZz??a 	  LL3di?@@@Jz**1-&&//v/QQQQQr&   c                     | j         S rX   )rm   r9   s    r$   r'   zLocalCatalogEntry.name   
    zr&   c           
         t          | j        t                    rd | j        D             }nkt          | j        t                    r$d | j                                        D             }n-t          | j        t
                    r| j        n| j        j        }| j        | j        ||| j	        | j
        d | j        D             | j        | j        d	S )"Basic information about this entryc                     g | ]	}|j         
S r    r'   r[   s     r$   r%   z.LocalCatalogEntry.describe.<locals>.<listcomp>   s    ///Q!&///r&   c                 4    i | ]\  }}|t          |          S r    )r   ri   s      r$   rg   z.LocalCatalogEntry.describe.<locals>.<dictcomp>   s$    CCCda!Yq\\CCCr&   c                 6    g | ]}|                                 S r    )rA   )r!   us     r$   r%   z.LocalCatalogEntry.describe.<locals>.<listcomp>   s     LLL

LLLr&   )	r'   rZ   pluginrf   r(   r   user_parametersr   r   )rC   ry   r{   r|   r}   rD   r'   rm   r   ro   rq   rt   ru   rr   )r#   pls     r$   rA   zLocalCatalogEntry.describe   s    dlD)) 	V//$,///BBd++ 	VCCdl.@.@.B.BCCCBB!+DL#!>!>UDLDUBJ,!0LLd6KLLLO

 

 
	
r&   c                    |                     dd           }| j        | j                                        ni }| j        |d<   |                     dd           s| j        r |                     dd           p| j        |d<   || j        d}|                    | j                   d|vr1| j        *| j        j        rt          |          r| j        j        |d<   t          ||| j        | j        | j        d          }t          | j                  dk    rt!          d	| j        z            t%          | j        t&                    r| j        d         }n|t'          | j                  d         }| j        |         }|                    |                    d
i                      	 | j        |         }n # t*          $ r t!          d|z            w xY w||fS )Nr   r   r   )r   CATALOG_DIRstorage_optionsF)rG   rF   rE   r   zNo plugins loaded for this entry: %s
A listing of installable plugins can be found at https://intake.readthedocs.io/en/latest/plugin-directory.html .r   zHAttempt to select unavailable plugin %s, perhaps import of plugin failed)popru   copyrv   rs   updaterr   rw   r   r   r   rt   rG   rF   r   ry   r0   rp   rC   r{   getKeyError)r#   r   r   mdparams	open_argsspecs          r$   _create_open_argsz#LocalCatalogEntry._create_open_args   s=    $$Xt44&*nHT^  """b -=w-- 	L 	L)--gt<<KBwK,
 
 	do&&&F* 	It/? 	IPTP`Pp 	I  vF  GM  vN  vN 	I(,(8(HF$%v8MX\Xenrny  CH  I  I  I	t|! 	y$ '+l3   d++ 	y\!_FF  /dl++A.<'DTXXfb11222yf- y y y !nqw!wxxxyy  s   'F5 5Gc                     |s| j         | j         S |                     |          \  }} |di |}| j        |_        | j        |_        | j        |_        | j        |_        |s|| _         |S )3Instantiate the DataSource for the given parametersNr    )rn   r   rx   catalog_objectr'   ro   r(   cat)r#   r   r   r   data_sources        r$   r   zLocalCatalogEntry.get  s     	(4#7 	('' 22?CC	f))y))%)]"9"&"3-  	/#.D r&   c                     d| _         dS )z
        Clear a cached default source so it can be created anew (if, for instance,
        it depends on changing environment variables or execution context)
        N)rn   r9   s    r$   clear_cached_default_sourcez-LocalCatalogEntry.clear_cached_default_source  s    
  $r&   )r*   rO   rP   rQ   r3   propertyr'   rA   r   r   r   __classcell__r8   s   @r$   rT   rT   }   s        -- 8<"B[]hjxz  DH  SW  aeKR KR KR KR KR KRZ   X
 
 
($! $! $!L  $$ $ $ $ $ $ $r&   rT   c                       e Zd ZdZddZed             Zed             Zed             Zed             Z	dd	Z
dd
Zd ZddZd Zd Zd Zd Zd ZdS )CatalogParserzLoads entries from a YAML specTNc                     |r|ni | _         g | _        g | _        || _        || _        |                     |          | _        d S rX   )_context_errors	_warningsrF   rG   _parse_data)r#   datarF   rG   contexts        r$   r3   zCatalogParser.__init__(  sE    #*2 [[&&


r&   c                 2    t          | j                  dk    S )Nr   )r   r   r9   s    r$   okzCatalogParser.ok0  s    4<  A%%r&   c                     | j         S rX   )r   r9   s    r$   r   zCatalogParser.data4  r   r&   c                     | j         S rX   )r   r9   s    r$   errorszCatalogParser.errors8  s
    |r&   c                     | j         S rX   )r   r9   s    r$   warningszCatalogParser.warnings<  s
    ~r&   c                     |,| j                             t          |||f                     d S | j                             t          ||f                     d S rX   )r   appendrD   r#   msgobjkeys       r$   errorzCatalogParser.error@  sZ     	1LS#sO 4 455555LS#J00000r&   c                     |+| j                             t          ||f                     d S | j                             t          |||f                     d S rX   )r   r   rD   r   s       r$   warningzCatalogParser.warningF  sZ     	8N!!#sCj//22222N!!#sCo"6"677777r&   c                 h   d|vrd S t          |d         t                    s|                     d|d           d S d|d         vr|                     d|d                    d S t          |d         d         t                    s|                     d|d         d           d S |d         d         D ]}t          |t                    s|                     d|d         d           5d|v rt	          |d                    Od|v r|                     d	|           j|                     d
|           d S )Npluginsz+value of key 'plugins' must be a dictionarysourcezmissing key 'source'z$value of key 'source' must be a listz5value in list of plugins sources must be a dictionarymoduledirzThe key 'dir', and in general the feature of registering plugins from a directory of Python scripts outside of sys.path, is no longer supported. Use 'module'.zmissing 'module')rC   r|   r   r{   register_plugin_module)r#   r   plugin_sources      r$   _parse_pluginszCatalogParser._parse_pluginsL  sw   D  	F$y/400 	JJDdIVVVF4	?* 	JJ-tI???F$y/(3T:: 	JJ=tIPXYYYF!)_X6 	> 	>MmT22 

UW[\eWfhpqqq=( 
>&}X'>????-' >

F "	    

-}====	> 	>r&   c                    ||v rt          ||         |          rF|r<||         |vr2|                     d                    ||         |          ||           n>||         S |                     d                    ||         |j                  ||           d S |r+|                     d                    |          |           d S |r|S |t          u rd n	 |            S )Nz&value '{}' is invalid (choose from {})z$value '{}' is not expected type '{}'zmissing required key '{}')rC   r   formatr*   object)r#   r   r   dtyperequiredr/   choicess          r$   _getitemzCatalogParser._getitemm  s   #: 	#c(E** n $s3xw6 $JJGNNsSVxY`aacfhklllls8O

AHHSSXSabbdgilmmm4 	JJ299#>>DDD4 	N3ttEEGG3r&   c                    t          t                    }||                     |dt                    |                     |dt          |          |                     |dt          d          |                     |dt          d          |                     |dt          d          |                     |d	t          d          d
}|d         |d         d S t          di |S )Nr(   r   )r   r/   Fr   r+   r,   r-   )r'   r(   r   r/   r+   r,   r-   r    )r{   r   r   rD   r   r   )r#   r'   r   valid_typesr   s        r$   _parse_user_parameterz#CatalogParser._parse_user_parameter  s    >** ==}cBBMM$[MII}}T9fu}MM==ufu=EE==ufu=EE}}T9fu}MM
 
 -  	F6N 	4&&v&&&r&   c                     |                     dd          rddlm}  |d| j        | j        d|S d|v rddlm}  ||          S |                     ||          S )	NremoteFr   )RemoteCatalogEntryrl   cls)remake_instancer    )r   intake.catalog.remoter   rF   rG   intake.utilsr   _parse_data_source_local)r#   r'   r   r   r   s        r$   _parse_data_sourcez CatalogParser._parse_data_source  s    88He$$ 		=@@@@@@%%YT[4=YYTXYYYd] 	=444444"?4(((00t<<<r&   c                 P   ||                      |dt          d          |                      |dt                    |                      |dt          ddg d          |                      |d	t          d          |                      |d
t          d          |                      |dt          d          d}|d         d S g |d<   d|v r:t          |d         t                    rt          j        t          |d         t                    s|                     d|d           d S |d         	                                D ]\  }}t          |t                    s1|                     d
                    |          |d         |           Kt          |t                    s1|                     d
                    |          |d         |           |                     ||          }|r|d                             |           t          d| j        d         | j        | j        d|S )Nr(   Fr   rf   r   forbid)r   allowforce)r   r/   r   r   r   r   )r'   r(   rf   r   r   r   r   r   z.value of key 'parameters' must be a dictionarykey '{}' must be a string&value of key '{}' must be a dictionaryroot)r   rF   rG   r    )r   rD   r   r|   r{   rC   r   ObsoleteParameterErrorr   r}   r   r   r   rT   r   rF   rG   )r#   r'   r   ds	parameterr   s         r$   r   z&CatalogParser._parse_data_source_local  s=   ==}cE=RRmmD(F;;!]]4#PU_g  rN  rN  rN]  O  OMM$uMEE]]4$]GGdJuMM
 
 h< 	4<4 	1$|,d33 8 77d<0$77 

KTS_```t#'#5#;#;#=#= 1 1i!$,, JJ:AA$GGlI[]abbb!)T22 JJJQQRVWWY]^jYkmqrrr00yAA 1|$++C000 uT]6-B4;aeanuurtuuur&   c                    g }d|vr|                      d|           |S t          |d         t                    rt          j        t          |d         t
                    s|                      d|d           |S |d                                         D ]\  }}t          |t                    s1|                      d                    |          |d         |           Kt          |t
                    s1|                      d                    |          |d         |           | 	                    ||          }|r|
                    |           |S )Nsourceszmissing key 'sources'z+value of key 'sources' must be a dictionaryr   r   )r   rC   r{   r   ObsoleteDataSourceErrorr|   r}   rD   r   r   r   )r#   r   r   r'   r   r   s         r$   _parse_data_sourcesz!CatalogParser._parse_data_sources  sX   D  	JJ.555Nd9ot,, 	544$y/400 	JJDdIVVVN O1133 	$ 	$LD&dC(( 

6==dCCT)_VZ[[[fd++ 

FMMdSSUYZcUdfjkkk))$77C $s###r&   c           	      D   t          |t                    s|                     d|           d S t          |                     |          |                     |          |                    di           |                    d          |                    d                    S )Nzcatalog must be a dictionaryr   r'   r(   )plugin_sourcesdata_sourcesr   r'   r(   )rC   r|   r   r   r   r   )r#   r   s     r$   r   zCatalogParser._parse  s    $%% 	JJ5t<<<F..t4411$77XXj"--&!!//
 
 
 	
r&   )TTNrX   )TNN)r*   rO   rP   rQ   r3   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r&   r$   r   r   %  s5       ((' ' ' ' & & X&   X   X   X1 1 1 18 8 8 8> > >B4 4 4 4$' ' '$
= 
= 
=%v %v %vN  :
 
 
 
 
r&   r   c                     t          |                                           D ]=\  }}|r6t          |t          t          f          r|d         }t          ||d           >dS )zFind plugins in given moduler   TclobberN)r
   r}   rC   r{   tupler	   )modrj   r@   s      r$   r   r     sp    (--3355 0 01 	0!dE]++ aDAq$////0 0r&   c                 j   d| v rBt          |           \  }}t          |                              |           }d|vr|dz   |z   }|S t          t          j                            t	          j                    t          j                            |                               } | d         dk    r| dz  } | S )Nz:///)	r   r   _parentr   ospathrK   getcwddirname)r  protocol_outs       r$   get_dirr
    s    } $T**!"8,,44T:: 	)U"S(C
27<<	RW__T5J5JKKLLDBx3 Kr&   c                   `     e Zd ZdZeZdZdZdZd fd	Z	ddZ
dd	Zd
 Zed             Z xZS )YAMLFileCatalogz*Catalog as described by a single YAML filer   Nyaml_file_catTc                 R   || _         |Qt                              d           t          j        dt
                     |                     |           d| _        n|| _        |                    dd          | _	        d|v| _
         t          t          |           j        di | dS )af  
        Parameters
        ----------
        path: str
            Location of the file to parse (can be remote)
        text: str (DEPRECATED)
            YAML contents of catalog, takes precedence over path
        autoreload : bool
            Whether to watch the source file for changes; make False if you want
            an editable Catalog
        Nz-YAMLFileCatalog `text` argument is deprecatedz`text` argument is deprecatedFfsr'   r    )r  r   r   r   warnDeprecationWarningparse
autoreloadr   
filesystemaccessr   r  r3   )r#   r  textr  kwargsr8   s        r$   r3   zYAMLFileCatalog.__init__  s     	 	)NNJKKKM9;MNNNJJt#DOO(DO **T400F*-ot$$-7777777r&   Fc                 N   | j         du rd| _        d| _         dS | j        s|r| j        pi }t	          | j        d          st	          | j        d          rD| j        }t          t          | j        dt          | j        dd                              | _        nf| j        >t          | j        fd	d
i|}t          |          dk    sJ |d         }|j        | _        n!| j                            | j        d
          }t          | j                  | _        |5 }|                                                                }ddd           n# 1 swxY w Y   d|v r0t"                              d           |                    dd          }|                     |           dS dS )zLoad text of catalog file and pass to parse

        Will do nothing if auto-reload is off and reload is not explicitly
        requested
        Fr   TNr  readr'   filemoderbr   )r  z
!template z&Use of '!template' deprecated - fixingrU   )r  updatedr  r   hasattrr  r   r=   r  r   r   r  openr
  _dirr  decoder   r   replacer  )r#   reloadoptions	file_openfr  s         r$   _loadzYAMLFileCatalog._load'  s    ;% 	DLDKF? 	f 	*0bGty&)) 	GWTY-G-G 	G I	+GDIvwtyZ`bhGiGi,j,jkk		 G&tyGGtGwGG	9~~****%aL	"+, O000FF		**DI )avvxx(() ) ) ) ) ) ) ) ) ) ) ) ) ) )t# 6GHHH||L"55JJt)	 	s   'EEEc                 ^   ddl }| j                                        }|p|j        pd}|||<   || j        pi }t          | j        gfddi|}n|pi }t          |gfddi|}t          |          dk    sJ |d         }| j        i d}	|D ]P}
t          ||
         
                                d                                                   d         |	d         |
<   Q|5 }|                    |	|d	
           ddd           n# 1 swxY w Y   |r| S t          | j        || j                  S )a9  Add sources to the catalog and save into the original file

        This adds the source into the catalog dictionary, and saves the
        resulting catalog as YAML. Typically, this would be used to update a
        catalog file in-place. Optionally, the new catalog can be saved to a
        new location, in which case the new catalog is returned.

        Note that if a source of the given name exists, it will be clobbered.

        Parameters
        ----------
        source : DataSource instance
            The source whose spec we want to save
        name : str or None
            The name the source is to have in the catalog; use the source's
            name attribute, if not given.
        path : str or None
            Location to save the new catalog; if None, the original location
            from which it was loaded
        storage_options : dict or None
            If saving to a new location, use these arguments for the filesystem
            backend

        Returns
        -------
        YAMLFileCatalog instance, containing the new entry
        r   Nr   r  wtr   )r   r   r   F)default_flow_style)r   r  )yaml_entriesr   r'   r   r   r  r   r   r{   _yamlr~   dumpr  r  )r#   r   r'   r  r   r+  entriesr$  r%  r   er&  s               r$   addzYAMLFileCatalog.addH  s   8 	-$$&&.v{.h 	A*0bG"DI;EETEWEEII%+G"D6@@@@@I9~~""""aL	 Mb99 	Q 	QA!%gaj&6&6&8&8&C&J&J&L&L!M!Ma!PDOA 	9!IIdA%I888	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9  	kK"49oZ^Zijjjjs   DDDc                    || _         t          | j                   }|t          j        d          t	          | j                  }t          ||| j        | j                  }|j	        rKt          j
        d                    | j        d                    |j	                            |j	                  |j        }i | _        |                    di                               di           }| j                            d	 |                                D                        |d
         D ]$}| |_        || j        |j        <   | j        |_        %| j                                        }|                    |                    di                      || _        | j        p|                    d          p| j        | _        | j        p|                    d          | _        dS )aL  Create entries from catalog text

        Normally the text comes from the file at self.path via the ``_load()``
        method, but could be explicitly set instead. A copy of the text is
        kept in attribute ``.text`` .

        Parameters
        ----------
        text : str
            YAML formatted catalog spec
        NzNo YAML data in file)r   )r   rF   rG   z'Catalog '{}' has validation errors:

{}
r   r   c                 0    i | ]\  }}|t          |fi |S r    )r   )r!   r'   attrss      r$   rg   z)YAMLFileCatalog.parse.<locals>.<dictcomp>  s/    $u$u$uKDRWT=+G+G+G+G$u$u$ur&   r   r'   r(   )r  r   r   CatalogExceptionr|   r   r   rF   rG   r   ValidationErrorr   r  rK   r   r,  r   r   r   r}   rx   r'   r  rw   r   r   name_from_pathr(   )	r#   r  r   r   resultcfgshared_parametersentrymetas	            r$   r  zYAMLFileCatalog.parse~  s    	## 	F-.DEEE DI&&&tWT[SWS`aaa= 	X,-[-b-bcgclnrnwnwx~  yF  oG  oG  .H  .H  JP  JW  X  X  Xk HHZ4488rJJ##$u$u[l[r[r[t[t$u$u$uvvv( 	0 	0E!EN(-DM%*% $E}!!##CGGJ++,,,IGGD4G	+Esww}/E/Er&   c                 J   t           j                            t           j                            | j                            d         }|dk    rAt           j                            t           j                            | j                            }|                    dd          S )z=If catalog is named 'catalog' take name from parent directoryr   r   .r  )r  r  splitextbasenamer  r"  )r#   r'   s     r$   r8  zYAMLFileCatalog.name_from_path  sw     w 0 0 ; ;<<Q?9 	@7##BGOODI$>$>??D||C%%%r&   )NNT)F)NNN)r*   rO   rP   rQ   r   versionrZ   partition_accessr'   r3   r'  r1  r  r   r8  r   r   s   @r$   r  r    s        44GID8 8 8 8 8 80   B4k 4k 4k 4kl'F 'F 'FR & & X& & & & &r&   r  c                   <     e Zd ZdZefZdZdZdZd fd	Z	d Z
 xZS )	YAMLFilesCatalogz-Catalog as described by a multiple YAML filesr   Nyaml_files_catTc                     || _         || _        |                                | _        g | _        i | _        d|v| _         t          t          |           j	        di | dS )a  
        Parameters
        ----------
        path: str
            Location of the files to parse (can be remote), including possible
            glob (*) character(s). Can also be list of paths, without glob
            characters.
        flatten: bool (True)
            Whether to list all entries in the cats at the top level (True)
            or create sub-cats from each file (False).
        r'   Nr    )
r  _flattenr   _kwargs
_cat_files_catsr  r   rE  r3   )r#   r  flattenr  r8   s       r$   r3   zYAMLFilesCatalog.__init__  sf     	{{}}
F*.%%.8888888r&   c                    | j         du rd| _        d| _         d S | j                                         | j        pi t          | j        t          t          f          rut          fd| j        D             g           }| j
        pdt          |          z  | _
        | j        pdt          |           d| _        d | j        D             | _        nt          | j        t                    rd	| j        vr| j        d
z   | _        t          | j        fddi}t          | j                  | _        | j
        p| j        | _
        | j        p	d| j         | _        t!          d |D                       t!          d | j        D                       k    s || _        | j                                         |D ]s}t&          j                            |j                  d                             dd                              dd          }| j                                        }|j        |d<   t          t&          j                            |j                            }|j        | j        vrt3          |d|z  dd|g g | j        |	  	        }| j        r	  |            }|                                 | j                            |j                   || j        |j        <   ,# t>          $ r+}t@          !                    d|d|           Y d }~\d }~ww xY w|| j        |j        <   ui }	t          | j        "                                          D ]\  }}| j        rj|                                 t!          |j                  #                    |	          }
|
rtI          d|
z            |	                    |j                   v||	|j%        <   | j                            |	           d S )NFr   Tc                 .    g | ]}t          |fd diS )r  r  )r   )r!   r\   r$  s     r$   r%   z*YAMLFilesCatalog._load.<locals>.<listcomp>  s.    PPPA<<D<G<<PPPr&   z%i fileszCatalog generated from z filesc                 ,    g | ]}t          |          S r    )r   r[   s     r$   r%   z*YAMLFilesCatalog._load.<locals>.<listcomp>  s     ???++???r&   *z/*r  r  z*Catalog generated from all files found in c              3   $   K   | ]}|j         V  d S rX   r  r!   r&  s     r$   r]   z)YAMLFilesCatalog._load.<locals>.<genexpr>  s$      ))a16))))))r&   c              3   $   K   | ]}|j         V  d S rX   rR  rS  s     r$   r]   z)YAMLFilesCatalog._load.<locals>.<genexpr>  s$      0Q0QA0Q0Q0Q0Q0Q0Qr&   r   z.yamlrU   z.ymlr  zYAML file: %sr  z	Loading "z" as a catalog failed: zVConflicting names when flattening multiple catalogs. Sources %s exist in more than one)&r  r  r,  clearr   rC   r  r{   r   sumr'   r   r(   rD   r   r   rz   rJ  rK  r  splitr"  r  r   r  rT   r   rH  r#  r   r   IOErrorr   infor}   intersectionr0   rm   )r#   filesr&  r'   r  ra   r<  r   r0  r/  interr$  s              @r$   r'  zYAMLFilesCatalog._load  s    ;% 	DLDKF&,"di$// 	lPPPPdiPPPRTUUE	<Z#e**%<DI#/_3_SQVZZ3_3_3_D??TY???DII$)S)) -c.B - I,	ty??t?w??E'	22DI	.TYDI#/k3k`d`i3k3kD))5)))))S0Q0Q0Q0Q0Q-Q-QQ 	#DOJ 	/ 	/A7==((,44WbAAII&RTUUD[%%''FVF6N 7 788AvTZ' /)$$0FY]_egikmoso|~  A  A= /\#egg

,33C4GHHH-0
16**" \ \ \QVQVQVXYXY$Z[[[[[[[[\ */DJqv&
 0 0 2 233 	- 	-KD%} -EN++88AA M$  &D  GL  &L  M  M  Mu~....',$$W%%%%%s   AK""
L, LL)T)r*   rO   rP   rQ   r   rB  rZ   rC  r'   r3   r'  r   r   s   @r$   rE  rE    si        77nGID9 9 9 9 9 9(7& 7& 7& 7& 7& 7& 7&r&   rE  c                   (     e Zd ZdZ fdZd Z xZS )MergedCatalogzB
    A Catalog that merges the entries of a list of catalogs.
    c                 H    || _          t                      j        |i | d S rX   )	_catalogsr   r3   )r#   catalogsr   r  r8   s       r$   r3   zMergedCatalog.__init__  s,    !$)&)))))r&   c                     | j         D ]}|                                 t          j        d | j         D              | _        d S )Nc              3   $   K   | ]}|j         V  d S rX   )r,  )r!   r   s     r$   r]   z&MergedCatalog._load.<locals>.<genexpr>  s%      .^.^Gw/?.^.^.^.^.^.^r&   )r`  r'  collectionsChainMapr,  )r#   r   s     r$   r'  zMergedCatalog._load  sF    ~ 	 	GMMOOOO#,.^.^t~.^.^.^_r&   r*   rO   rP   rQ   r3   r'  r   r   s   @r$   r^  r^    sX         * * * * *` ` ` ` ` ` `r&   r^  c                   J     e Zd ZdZ fdZd Zed             Zd Zd Z	 xZ
S )EntrypointEntryz-
    A catalog entry for an entrypoint.

    c                 r    || _         d | _        g | _        t                                                       d S rX   )_entrypointr   rt   r   r3   )r#   
entrypointr8   s     r$   r3   zEntrypointEntry.__init__  s6    % "r&   c                     d| j          dS )Nz<Entry 'z'>r   r9   s    r$   r:   zEntrypointEntry.__repr__!  s    '$)''''r&   c                     | j         j        S rX   )rj  r'   r9   s    r$   r'   zEntrypointEntry.name$  s    $$r&   c                     | j          |             j        | _         | j        | j        j        | j        j        | j        j        | j        j        | j         dS )r   N)r'   module_nameobject_namedistroextrasrZ   )r   rZ   r'   rj  ro  rp  rq  rr  r9   s    r$   rA   zEntrypointEntry.describe(  sY    ? 	/"dff.DOI+7+7&-&-
 
 	
r&   c                 4    | j                                         S )r   )rj  loadr9   s    r$   r   zEntrypointEntry.get5  s    $$&&&r&   )r*   rO   rP   rQ   r3   r:   r   r'   rA   r   r   r   s   @r$   rh  rh    s         
    ( ( ( % % X%
 
 
' ' ' ' ' ' 'r&   rh  c                   0     e Zd ZdZddd fd
Zd Z xZS )EntrypointsCatalogz6
    A catalog of discovered entrypoint catalogs.
    zintake.catalogsN)entrypoints_grouppathsc                V    || _         || _         t                      j        |i | d S rX   )_entrypoints_group_pathsr   r3   )r#   rw  rx  r   r  r8   s        r$   r3   zEntrypointsCatalog.__init__?  s4    "3$)&)))))r&   c                 n   t          j        | j        | j                  }| j        pd| _        | j        pdt          |           d| _        |                                D ]R\  }}	 t          |          | j	        |<   # t          $ r(}t          j        d| d| d|d           Y d }~Kd }~ww xY wd S )NrR  rv  zEntrypointsCatalog of z
 catalogs.zFailed to load z, r?  )entrypointsget_group_namedrz  r{  r'   r(   r   r}   rh  r,  	Exceptionr   r  )r#   ra  r'   rk  r0  s        r$   r'  zEntrypointsCatalog._loadD  s    .t/FT[YYYI5!5	+a/aH/a/a/a ( 0 0 	N 	ND*N&5j&A&Ad## N N NLLL
LLaLLLMMMMMMMMN	N 	Ns   (B  
B2
B--B2rf  r   s   @r$   rv  rv  :  sj          1B * * * * * * *
N N N N N N Nr&   rv  r  Tr   rF  )/rd  r   loggingr  r   r}  fsspecr   r   fsspec.corer   rU   r   r   r   r	   source.discoveryr
   utilsr   r   r   r   r   baser   r   r<  r   r   r   r   r   r   	getLoggerr   r   rT   r   r   r   r
  r  rE  r^  rh  rv  r    r&   r$   <module>r     s*         				      3 3 3 3 3 3 3 3 & & & & & &       6 6 6 6 6 6 6 6 7 7 7 7 7 7 M M M M M M M M M M M M       % % % % % % % %       X X X X X X X X X X X X X X		8	$	$] ] ] ] ]& ] ] ]@e$ e$ e$ e$ e$ e$ e$ e$PG
 G
 G
 G
 G
F G
 G
 G
T0 0 0  f& f& f& f& f&g f& f& f&RS& S& S& S& S&w S& S& S&l` ` ` ` `G ` ` `"' "' "' "' "'l "' "' "'JN N N N N N N N0 $ ? ? ? ?  "2D A A A A A Ar&   