
    +d'O                     :   d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlm	Z	m
Z
 ddlmZmZ ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZmZmZ  ej        d          Z G d de          Z G d de j        j                  Z G d de          Z ddZ!dS )    N)AuthenticationFailureBaseClientAuth   )pack_kwargsunpack_kwargs)registry)remake_instance   )Catalog)CatalogEntry)RemoteCatalogErrorcoerceexpand_defaultsintakec                        e Zd ZdZdZ	 	 	 	 	 	 	 	 	 	 	 	 d fd	Zd Zd	 Zd
 Ze	d             Z
d Zd Zd Zd Zd Zd Zed             Zed             Z xZS )RemoteCatalogz#The state of a remote Intake serverintake_remoteNr
   Tdefaultc           	          ddl m}m} |i }nt          j        |          }|                    dd          }|rdnd}|                    d|d	          }|                    d
          s|d
z   }|| _         ||d          | _	         ||d          | _
        || _        | j                            |pi            | j                            di           | j        d<   || _        || _        || _        d| _        |pt%                      | _        | j        8 ||          j                            dd                              dd          }t+          t,          |                               ||||	|
||           dS )ad  Connect to remote Intake Server as a catalog

        Parameters
        ----------
        url: str
            Address of the server, e.g., "intake://localhost:5000".
        http_args: dict
            Arguments to add to HTTP calls, including "ssl" (True/False) for
            secure connections.
        page_size : int, optional
            The number of entries fetched at a time during iteration.
            Default is None (no pagination; fetch all entries in bulk).
        name : str, optional
            Unique identifier for catalog. This is primarily useful when
            manually constructing a catalog. Defaults to None.
        source_id : str, optional
            Emphemeral unique ID generated by the server, if known.
        metadata: dict
            Additional information about this data
        auth : BaseClientAuth or None
            Default, None, falls back to BaseClientAuth.
        ttl : float, optional
            Lifespan (time to live) of cached modification time. Units are in
            seconds. Defaults to 1.
        getenv: bool
            Can parameter default fields take values from the environment
        getshell: bool
            Can parameter default fields run shell commands
        storage_options : dict
            If using a URL beginning with 'intake://' (remote Intake server),
            parameters to pass to requests when issuing http commands; otherwise
            parameters to pass to remote backend file-system. Ignored for
            normal local files.
        parameters: dict
            To pass to the server when it instantiates the data source
        r   )urljoinurlparseNsslFhttpshttpr   r
   /zv1/info	v1/sourceheaders._:)namemetadatattlgetenvgetshellstorage_optionspersist_mode)requests.compatr   r   copydeepcopypopreplaceendswithurlinfo_url
source_url	http_argsupdateget
_page_size
_source_id_parameters_lenr   authnetlocsuperr   __init__)selfr.   r1   	page_sizer!   	source_idr"   r8   r#   r$   r%   r&   
parametersr'   r   r   securescheme	__class__s                     5lib/python3.11/site-packages/intake/catalog/remote.pyr;   zRemoteCatalog.__init__#   s   h 	65555555 	1II i00Iue,,".kk(FA..||C   	)CY//!'#{33"o3444$(N$6$6y"$E$Ey!##%	,N,,	? 	L8C=='//S99AA#sKKDmT""++#fxap  @L 	, 	
 	
 	
 	
 	
    c                      t          |           S N)Entriesr<   s    rC   _make_entries_containerz%RemoteCatalog._make_entries_containeru   s    t}}rD   c                     d |                                  D             t          | j                                                  z   S )Nc                 d    g | ]-}t          j        d |          t          j        |          +|.S )z[_A-Za-z][_a-zA-Z0-9]*$)rematchkeyword	iskeyword).0keys     rC   
<listcomp>z)RemoteCatalog.__dir__.<locals>.<listcomp>z   s_     
 
 
rxHacf?g?g
pw  qB  CF  qG  qG

 
 
rD   )_ipython_key_completions_list__dict__keysrH   s    rC   __dir__zRemoteCatalog.__dir__x   sR    
 
99;;
 
 
M  
 

 	
rD   c                     | j         j        st          t          |                      | j         j        st	          j        d           d | j                                         D             S )Nz[Tab-complete and dir() on RemoteCatalog may include only a subset of the available entries.c                     g | ]\  }}|S  rZ   )rP   rQ   r   s      rC   rR   z;RemoteCatalog._ipython_key_completions_.<locals>.<listcomp>   s    ???Q???rD   )_entriescompletenextiterwarningswarncached_itemsrH   s    rC   rS   z'RemoteCatalog._ipython_key_completions_   si    }% 	 d}% 	|Mz{{{ @?$-"<"<">">????rD   c                     | j         S rF   )r4   rH   s    rC   r=   zRemoteCatalog.page_size   s
    rD   c                 4   dd l }t                              d||| j        z              || j        d}|                     |          } |j        | j        fi |}	 |                                 n># |j        $ r1}t          d
                    ||| j        z                       |d }~ww xY wt          j        |j        fi t          }i }|d         D ]B}	t          d| j        | j        | j        | j        | j        | j        | j        d|	||	d         <   C|S )	Nr   zRequest page entries %d-%dpage_offsetr=   z&Failed to fetch page of entries {}-{}.sourcesr.   r$   r%   r8   r1   r=   r'   r!   rZ   )requestsloggerdebugr4   _get_http_argsr3   r/   raise_for_status	HTTPErrorr   formatmsgpackunpackbcontentr   RemoteCatalogEntryr.   r$   r%   r8   r1   pmode)
r<   re   rh   paramsr1   responseerrinfopagesources
             rC   
fetch_pagezRemoteCatalog.fetch_page   s|   1;do@]^^^!,4?KK''//	8<;;;;	N%%''''! 	N 	N 	N$%P%W%WXcepsw  tC  fC  &D  &D  E  E  KN  N	Nx/AA=AA9o 	 	F $6 
$H{Y./!Z
$ 
$ 
$ 
$D   s   A1 1
B,;,B''B,c                    dd l }t                              d|           d|i}|                     |          } |j        | j        fi |}|j        dk    rt          |          	 |                                 n5# |j	        $ r(}t          d                    |                    |d }~ww xY wt          j        |j        fi t          }t!          d| j        | j        | j        | j        | j        | j        | j        d|d         S )	Nr   z&Requesting info about entry named '%s'r!   i  zFailed to fetch entry {!r}.rg   ry   rZ   )rh   ri   rj   rk   r3   r0   status_codeKeyErrorrl   rm   r   rn   ro   rp   rq   r   rr   r.   r$   r%   r8   r1   r4   rs   )r<   r!   rh   rt   r1   ru   rv   rw   s           rC   fetch_by_namezRemoteCatalog.fetch_by_name   s;   =tDDD$''//	8<==9==3& 	!4.. 	Z%%''''! 	Z 	Z 	Z$%B%I%I$%O%OPPVYY	Zx/AA=AA! 	
;]no	
 	
 8n	
 	
 		
s   'A< <
B.#B))B.c                 d   | j                             di           }| j        .| j                                        }|                    |           | j                                         }| j        
| j        |d<   ||d<   |                    di           }|                    |           ||d<   |S )zn
        Return a copy of the http_args

        Adds auth headers and 'source-id', merges in params.
        r   Nz	source-idrt   )r1   r3   r8   get_headersr2   r)   r5   )r<   rt   r   auth_headersr1   merged_paramss         rC   rk   zRemoteCatalog._get_http_args   s     .$$Y339 	)90022LNN<((( N''))	? 	3#'?GK &	) "h33V$$$+	(rD   c                 ~    ddl } j        i }nddd}                     |          } |j         j        fi |}	 |                                 d}n9# |j        $ r,}d|j        d         v rd}nt          d          |Y d}~nd}~ww xY w|rt          |          t          j        |j        fi t          }|d          _        |                    d	           _         j                                         |d
         r:d _         j        j                             fd|d
         D                        dS dS )z7Fetch metadata from remote. Entries are fetched lazily.r   Nrd   F403z9Your current level of authentication does not have accesszFailed to fetch metadata.r"   lengthrf   c                 t    i | ]4}|d          t          dj        j        j        j        j        d|5S )r!   )r.   r$   r%   r8   r1   rZ   )rr   r.   r$   r%   r8   r1   )rP   ry   r<   s     rC   
<dictcomp>z'RemoteCatalog._load.<locals>.<dictcomp>	  sw        6N$6  %e48DKbfbovzv  LP  LZ  %e  %e  ^d  %e  %e  rD   )rh   r=   rk   r3   r/   rl   rm   argsr   r   ro   rp   rq   r   r"   r7   r[   resetr4   _page_cacher2   )r<   rh   rt   r1   ru   errorrv   rw   s   `       rC   _loadzRemoteCatalog._load   s    	> 	8FF &'Q77F''//	8<;;;;	O%%'''EE! 	O 	O 	O# OS()DEE3N 	O
  	/'...x/AA=AAZ( HHX&&	 	? 	"DOM%,,   "&y/      	 	s   A 
B"BBc           
         dd l }d||f| j        d} |j        d
d| j        i|                     i           dt          j        |fi t          i}	 |                                 n"# |j	        $ r}t          d          |d }~ww xY wt          j        |j        fi t          }|d         }t          | j        | j        || j        d	          }	| |	_        |	S )Nr   search)actionqueryr>   r.   datazFailed search query.r>    )r.   r1   r>   r'   r!   rZ   )rh   r5   postr0   rk   ro   packbr   rl   rm   r   rp   rq   r   r   r.   r1   rs   cat)
r<   r   kwargsrh   requestru   rv   ry   r>   r   s
             rC   r   zRemoteCatalog.search  s   %vT_]] 8=||T_|8K8KB8O8O||V]VcdkV{V{ozV{V{|||	F%%''''! 	F 	F 	F$%;<<#E	F!1CC]CC;'	DNifjfpwyzzz
s   A( (
B2BBc                 P    | j         t          d | D                       S | j         S )Nc              3      K   | ]}d V  dS )r
   NrZ   )rP   r   s     rC   	<genexpr>z(RemoteCatalog.__len__.<locals>.<genexpr>#  s"      ''Qq''''''rD   )r7   sumrH   s    rC   __len__zRemoteCatalog.__len__  s3    9 	 ''$''''''9rD   c                 (    t          j        | |fi |S rF   )r   _data_to_source)ry   pathr   s      rC   _persistzRemoteCatalog._persist'  s    ,VTDDVDDDrD   c                    dd l }ddlm} ddlm} t          | t                    st          i }| j        	                                D ]W\  }}|
                                ||<   d |j        D             ||         d<   ||         d                             d           Xt          j        |d          }	 ||	gd          d         5 }
|                    d	|i|
           d d d            n# 1 swxY w Y    ||	          S )
Nr   )
open_files)YAMLFileCatalogc                     g | ]	}|j         
S rZ   )_captured_init_kwargsrP   ups     rC   rR   z1RemoteCatalog._data_to_source.<locals>.<listcomp>9  s    &a&a&aBr'?&a&a&arD   r?   r   zcat.yamlwtrf   )yamlfsspecr   intake.catalog.localr   
isinstancer   NotImplementedErrorr[   items__getstate___user_parametersr+   	posixpathjoindump)r   r   r   r   r   r   outr!   entryfnfs              rC   r   zRemoteCatalog._data_to_source+  sh   %%%%%%888888#w'' 	&%% <--// 	2 	2KD%**,,CI&a&a%J`&a&a&aCIl#Ih##L1111^D*--Zd##A& 	+!IIy#&***	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+r"""s   C--C14C1)NNNNNNr
   TTNNr   )__name__
__module____qualname____doc__r!   r;   rI   rW   rS   propertyr=   rz   r~   rk   r   r   r   staticmethodr   r   __classcell__rB   s   @rC   r   r      sT       --D
 P
 P
 P
 P
 P
 P
d  
 
 

@ 
@ 
@   X  >
 
 
2  0- - -^     E E \E # # \# # # # #rD   r   c                   6    e Zd ZdZd Zd Zd Zd Zd Zd Z	dS )	rG   zFetches entries from server on item lookup and iteration.

    This fetches pages of entries from the server during iteration and
    caches them. On __getitem__ it fetches the specific entry from the
    server.
    c                     || _         t          j                    | _        i | _        d| _        | j         j        d u | _        d S )Nr   )_catalogcollectionsOrderedDictr   _direct_lookup_cache_page_offsetr=   r\   )r<   catalogs     rC   r;   zEntries.__init__K  sA    &244 %'!/47rD   c                     | j                                          | j                                         d| _        | j        j        du | _        dS )zClear caches to force a reload.r   N)r   clearr   r   r   r=   r\   rH   s    rC   r   zEntries.resetV  sI       !'')))/47rD   c              #   L  K   | j         D ]}|V  | j        j        d S 	 | j                            | j                  }| j                             |           | xj        t          |          z  c_        |D ]}|V  t          |          | j        j        k     r	d| _        d S )NT)r   r   r=   rz   r   r2   lenr\   )r<   rQ   rx   s      rC   __iter__zEntries.__iter__]  s      # 	 	CIIII=" 	 F	=++D,=>>D##D)))T*  				4yy4=22  !%	rD   c              #      K   | j                                         D ]}|V  | j                                        D ]}|V  dS )zR
        Iterate over items that are already cached. Perform no requests.
        N)r   r   r   )r<   items     rC   ra   zEntries.cached_itemss  sd       $**,, 	 	DJJJJ-3355 	 	DJJJJ	 	rD   c                     	 | j         |         S # t          $ rI 	 | j        |         cY S # t          $ r+ | j                            |          }|| j         |<   |cY cY S w xY ww xY wrF   )r   r}   r   r   r~   )r<   rQ   ry   s      rC   __getitem__zEntries.__getitem__|  s    	,S11 	 	 	',,,,   44S9917)#.	s)    
A")A"0AA"AA"c                 *    t          | j                  S rF   )r   r   rH   s    rC   r   zEntries.__len__  s    4=!!!rD   N)
r   r   r   r   r;   r   r   ra   r   r   rZ   rD   rC   rG   rG   A  sx         	8 	8 	88 8 8  ,  	 	 	" " " " "rD   rG   c                   F     e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 d
 fd	Zd Zd	 Z xZS )rr   z.An entry referring to a remote data definitionNr   r   FTc                    || _         t          |t                    rt          |          }|| _        || _        || _        || _        |pi | _        |	| _	        |
| _
        d |pg D             | _        || _        |pi                                 | _        d| j        vr
i | j        d<   t          t           |                               ||           d| _        dS )aj  

        Parameters
        ----------
        url: str
            HTTP address of the Intake server this entry comes from
        auth: Auth instance
            If there are additional headers to add to calls, this instance will
            provide them
        kwargs: additional keys describing the entry, name, description,
            container,
        c                 b    g | ],}t          |t                    rd |v rt          |          n|-S )cls)r   dictr	   r   s     rC   rR   z/RemoteCatalogEntry.__init__.<locals>.<listcomp>  sT      !H  !H  !HkmB9M9M!fRW[]R]!f!4!4!4df  !H  !H  !HrD   r   )r$   r%   neverN)r.   r   r   r	   r8   	containerr!   description	_metadatar4   catalog_pmoder   _direct_accessr)   r1   r:   rr   r;   _pmode)r<   r.   r8   r!   user_parametersr   r   r"   r1   r=   r'   direct_accessr$   r%   r   rB   s                  rC   r;   zRemoteCatalogEntry.__init__  s   : dD!! 	)"4((D	"	&!R#) !H  !H  rA  rG  EG  !H  !H  !H+#/r//11DN* 	+(*DN9% $''000RRR rD   c           	      `    | j         | j        d| j        | j        | j        | j        | j        fdS )Nremote)r!   r   pluginr   r   r"   r   r   )r!   r   r   r   r   r   r.   rH   s    rC   describezRemoteCatalogEntry.describe  s<    I+!0#4XK	
 	
 		
rD   c                 N   | j         D ]j}|d         |vr^|d         }t          |t                    r6t          |d         t	          |d         d| j        | j                            }|||d         <   k| j                                        }| j        d                                         |d<   |d         	                    | j
                                                   t          | j        | j        | j        || j        || j        | j
        | j        | j        | j                  S )Nr!   r   typeTr   )	r   r   r   r1   r=   r8   r$   r'   r%   )r   r   strr   r   r$   r%   r1   r)   r2   r8   r   open_remoter.   r!   r   r   r4   r   )r<   r   parr   r1   s        rC   r3   zRemoteCatalogEntry.get  s!   ( 	7 	7C6{/1 7i.gs++ u$S[/#i.RVX\Xceier2s2sttG/6F,N''))	#~i8==??	))##DI$9$9$;$;<<<HIn+(o;+]
 
 
 	
rD   )NNNr   NNNr   FTT)r   r   r   r   r;   r   r3   r   r   s   @rC   rr   rr     s        88 1 1 1 1 1 1f

 

 


 
 
 
 
 
 
rD   rr   c           	         ddl }ddl}ddlm} ddlm} |                     d          r| t          d          d         } t          d||t          t                              } |j         || d          fd	 |j        |fi t          i|}|j        r |j        |j        fi t"          }d
|v rO|d
         }t%          |t&                    r|gn|}|D ]'}|t          v rt          |         di |d         }d} n(d}nd}|r^|                    d           |                    ||d           |dk    r|                    ||	|
||d            ||         | |fi |}||_        |S t/          d|j        |j        fz            )z@Create either local direct data source or remote streamed sourcer   N)r   )container_mapz	intake://open)r   r!   r?   available_pluginsr   r   r   r   FTr   )r!   r?   r   )r8   r$   r%   r=   r'   zServer error: %d, %srZ   )ro   rh   r(   r   intake.containerr   
startswithr   r   rT   plugin_registryr   r   r   okrp   rq   r   r   r   r+   r2   r   	Exceptionr|   reason)r.   r   r   r   r   r1   r=   r'   r8   r$   r%   ro   rh   r   r   payloadreqru   plpry   proxys                         rC   r   r     s+   NNNOOO''''''......
~~k"" &#k""$$%&u\`ap\q\qrrrG
(-[11
k
kg8]8]Q\8]8]
kaj
k
kC
v #P"7?3;@@-@@x 	(#B#B,,4""B  ' ,Q/CC(62BCCF!EE
 E 	JLL%%%OOU/JJKKKI%  $"($,%.(4 
 
 
 .]9-c9IIIIF(.#/3:1NNOOOrD   )NNNNN)"r   r)   rN   loggingr   rL   r_   ro   intake.auth.baser   r   compatr   r   ry   r   r   utilsr	   r   r   r   r   r   r   r   	getLoggerri   r   abcMappingrG   rr   r   rZ   rD   rC   <module>r      s              				   B B B B B B B B / / / / / / / / 0 0 0 0 0 0 # # # # # #             > > > > > > > > > >		8	$	$`# `# `# `# `#G `# `# `#F	G" G" G" G" G"ko% G" G" G"TY
 Y
 Y
 Y
 Y
 Y
 Y
 Y
x/P /P /P /P /P /PrD   